Skip to content

A
angle
Commit ba2cfe7c by Alok Priyadarshi
Options

Moved the definition of max-token length from preprocessor to compiler.

R=kbr@chromium.org Review URL: https://codereview.appspot.com/12957043
parent 76985f3c
Showing with 634 additions and 621 deletions
src/compiler/glslang.l
...@@ -321,6 +321,7 @@ int glslang_scan(size_t count, const char* const string[], const int length[], ...@@ -321,6 +321,7 @@ int glslang_scan(size_t count, const char* const string[], const int length[],
// Initialize preprocessor. // Initialize preprocessor.
if (!context->preprocessor.init(count, string, length)) if (!context->preprocessor.init(count, string, length))
return 1; return 1;
context->preprocessor.setMaxTokenLength(SH_MAX_TOKEN_LENGTH);
// Define extension macros. // Define extension macros.
const TExtensionBehavior& extBehavior = context->extensionBehavior(); const TExtensionBehavior& extBehavior = context->extensionBehavior();
......
src/compiler/glslang.y
...@@ -51,6 +51,7 @@ WHICH GENERATES THE GLSL ES PARSER (glslang_tab.cpp AND glslang_tab.h). ...@@ -51,6 +51,7 @@ WHICH GENERATES THE GLSL ES PARSER (glslang_tab.cpp AND glslang_tab.h).
%code requires { %code requires {
#define YYLTYPE TSourceLoc #define YYLTYPE TSourceLoc
#define YYLTYPE_IS_DECLARED 1 #define YYLTYPE_IS_DECLARED 1
#define SH_MAX_TOKEN_LENGTH 256 // WebGL spec.
} }
%union { %union {
......
src/compiler/glslang_lex.cpp
...@@ -2930,6 +2930,7 @@ int glslang_scan(size_t count, const char* const string[], const int length[], ...@@ -2930,6 +2930,7 @@ int glslang_scan(size_t count, const char* const string[], const int length[],
// Initialize preprocessor. // Initialize preprocessor.
if (!context->preprocessor.init(count, string, length)) if (!context->preprocessor.init(count, string, length))
return 1; return 1;
context->preprocessor.setMaxTokenLength(SH_MAX_TOKEN_LENGTH);
// Define extension macros. // Define extension macros.
const TExtensionBehavior& extBehavior = context->extensionBehavior(); const TExtensionBehavior& extBehavior = context->extensionBehavior();
......
src/compiler/glslang_tab.cpp
...@@ -126,6 +126,7 @@ extern int yydebug; ...@@ -126,6 +126,7 @@ extern int yydebug;
#define YYLTYPE TSourceLoc #define YYLTYPE TSourceLoc
#define YYLTYPE_IS_DECLARED 1 #define YYLTYPE_IS_DECLARED 1
#define SH_MAX_TOKEN_LENGTH 256 // WebGL spec.
...@@ -713,27 +714,27 @@ static const yytype_int16 yyrhs[] = ...@@ -713,27 +714,27 @@ static const yytype_int16 yyrhs[] =
/* YYRLINE[YYN] -- source line where rule number YYN was defined. */ /* YYRLINE[YYN] -- source line where rule number YYN was defined. */
static const yytype_uint16 yyrline[] = static const yytype_uint16 yyrline[] =
{ {
0, 179, 179, 180, 183, 226, 229, 242, 247, 252, 0, 180, 180, 181, 184, 227, 230, 243, 248, 253,
258, 261, 264, 267, 362, 372, 385, 393, 493, 496, 259, 262, 265, 268, 363, 373, 386, 394, 494, 497,
504, 507, 513, 517, 524, 530, 539, 547, 602, 612, 505, 508, 514, 518, 525, 531, 540, 548, 603, 613,
615, 625, 635, 656, 657, 658, 663, 664, 672, 683, 616, 626, 636, 657, 658, 659, 664, 665, 673, 684,
684, 692, 703, 707, 708, 718, 728, 738, 751, 752, 685, 693, 704, 708, 709, 719, 729, 739, 752, 753,
762, 775, 779, 783, 787, 788, 801, 802, 815, 816, 763, 776, 780, 784, 788, 789, 802, 803, 816, 817,
829, 830, 847, 848, 861, 862, 863, 864, 865, 869, 830, 831, 848, 849, 862, 863, 864, 865, 866, 870,
872, 883, 891, 918, 923, 937, 992, 995, 1002, 1010, 873, 884, 892, 919, 924, 938, 993, 996, 1003, 1011,
1031, 1052, 1062, 1090, 1095, 1105, 1110, 1120, 1123, 1126, 1032, 1053, 1063, 1091, 1096, 1106, 1111, 1121, 1124, 1127,
1129, 1135, 1142, 1145, 1167, 1185, 1209, 1232, 1236, 1254, 1130, 1136, 1143, 1146, 1168, 1186, 1210, 1233, 1237, 1255,
1262, 1294, 1314, 1335, 1344, 1367, 1370, 1376, 1384, 1392, 1263, 1295, 1315, 1336, 1345, 1368, 1371, 1377, 1385, 1393,
1400, 1410, 1417, 1420, 1423, 1429, 1432, 1447, 1451, 1455, 1401, 1411, 1418, 1421, 1424, 1430, 1433, 1448, 1452, 1456,
1459, 1463, 1468, 1473, 1478, 1483, 1488, 1493, 1498, 1503, 1460, 1464, 1469, 1474, 1479, 1484, 1489, 1494, 1499, 1504,
1508, 1513, 1518, 1523, 1527, 1531, 1539, 1547, 1551, 1564, 1509, 1514, 1519, 1524, 1528, 1532, 1540, 1548, 1552, 1565,
1564, 1578, 1578, 1587, 1590, 1606, 1639, 1643, 1649, 1656, 1565, 1579, 1579, 1588, 1591, 1607, 1640, 1644, 1650, 1657,
1671, 1675, 1679, 1680, 1686, 1687, 1688, 1689, 1690, 1694, 1672, 1676, 1680, 1681, 1687, 1688, 1689, 1690, 1691, 1695,
1695, 1695, 1695, 1705, 1706, 1710, 1710, 1711, 1711, 1716, 1696, 1696, 1696, 1706, 1707, 1711, 1711, 1712, 1712, 1717,
1719, 1729, 1732, 1738, 1739, 1743, 1751, 1755, 1765, 1770, 1720, 1730, 1733, 1739, 1740, 1744, 1752, 1756, 1766, 1771,
1787, 1787, 1792, 1792, 1799, 1799, 1807, 1810, 1816, 1819, 1788, 1788, 1793, 1793, 1800, 1800, 1808, 1811, 1817, 1820,
1825, 1829, 1836, 1843, 1850, 1857, 1868, 1877, 1881, 1888, 1826, 1830, 1837, 1844, 1851, 1858, 1869, 1878, 1882, 1889,
1891, 1897, 1897 1892, 1898, 1898
}; };
#endif #endif
......
src/compiler/glslang_tab.h
...@@ -44,6 +44,7 @@ extern int yydebug; ...@@ -44,6 +44,7 @@ extern int yydebug;
#define YYLTYPE TSourceLoc #define YYLTYPE TSourceLoc
#define YYLTYPE_IS_DECLARED 1 #define YYLTYPE_IS_DECLARED 1
#define SH_MAX_TOKEN_LENGTH 256 // WebGL spec.
......
src/compiler/preprocessor/Preprocessor.cpp
// //
// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved. // Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. // found in the LICENSE file.
// //
#include "Preprocessor.h" #include "Preprocessor.h"
#include <cassert> #include <cassert>
#include <sstream> #include <sstream>
#include "DiagnosticsBase.h" #include "DiagnosticsBase.h"
#include "DirectiveParser.h" #include "DirectiveParser.h"
#include "Macro.h" #include "Macro.h"
#include "MacroExpander.h" #include "MacroExpander.h"
#include "Token.h" #include "Token.h"
#include "Tokenizer.h" #include "Tokenizer.h"
namespace pp namespace pp
{ {
struct PreprocessorImpl struct PreprocessorImpl
{ {
Diagnostics* diagnostics; Diagnostics* diagnostics;
MacroSet macroSet; MacroSet macroSet;
Tokenizer tokenizer; Tokenizer tokenizer;
DirectiveParser directiveParser; DirectiveParser directiveParser;
MacroExpander macroExpander; MacroExpander macroExpander;
PreprocessorImpl(Diagnostics* diag, PreprocessorImpl(Diagnostics* diag,
DirectiveHandler* directiveHandler) : DirectiveHandler* directiveHandler) :
diagnostics(diag), diagnostics(diag),
tokenizer(diag), tokenizer(diag),
directiveParser(&tokenizer, &macroSet, diag, directiveHandler), directiveParser(&tokenizer, &macroSet, diag, directiveHandler),
macroExpander(&directiveParser, &macroSet, diag) macroExpander(&directiveParser, &macroSet, diag)
{ {
} }
}; };
Preprocessor::Preprocessor(Diagnostics* diagnostics, Preprocessor::Preprocessor(Diagnostics* diagnostics,
DirectiveHandler* directiveHandler) DirectiveHandler* directiveHandler)
{ {
mImpl = new PreprocessorImpl(diagnostics, directiveHandler); mImpl = new PreprocessorImpl(diagnostics, directiveHandler);
} }
Preprocessor::~Preprocessor() Preprocessor::~Preprocessor()
{ {
delete mImpl; delete mImpl;
} }
bool Preprocessor::init(size_t count, bool Preprocessor::init(size_t count,
const char* const string[], const char* const string[],
const int length[]) const int length[])
{ {
static const int kGLSLVersion = 100; static const int kGLSLVersion = 100;
// Add standard pre-defined macros. // Add standard pre-defined macros.
predefineMacro("__LINE__", 0); predefineMacro("__LINE__", 0);
predefineMacro("__FILE__", 0); predefineMacro("__FILE__", 0);
predefineMacro("__VERSION__", kGLSLVersion); predefineMacro("__VERSION__", kGLSLVersion);
predefineMacro("GL_ES", 1); predefineMacro("GL_ES", 1);
return mImpl->tokenizer.init(count, string, length); return mImpl->tokenizer.init(count, string, length);
} }
void Preprocessor::predefineMacro(const char* name, int value) void Preprocessor::predefineMacro(const char* name, int value)
{ {
std::ostringstream stream; std::ostringstream stream;
stream << value; stream << value;
Token token; Token token;
token.type = Token::CONST_INT; token.type = Token::CONST_INT;
token.text = stream.str(); token.text = stream.str();
Macro macro; Macro macro;
macro.predefined = true; macro.predefined = true;
macro.type = Macro::kTypeObj; macro.type = Macro::kTypeObj;
macro.name = name; macro.name = name;
macro.replacements.push_back(token); macro.replacements.push_back(token);
mImpl->macroSet[name] = macro; mImpl->macroSet[name] = macro;
} }
void Preprocessor::lex(Token* token) void Preprocessor::setMaxTokenLength(size_t maxLength)
{ {
bool validToken = false; mImpl->tokenizer.setMaxTokenLength(maxLength);
while (!validToken) }
{
mImpl->macroExpander.lex(token); void Preprocessor::lex(Token* token)
switch (token->type) {
{ bool validToken = false;
// We should not be returning internal preprocessing tokens. while (!validToken)
// Convert preprocessing tokens to compiler tokens or report {
// diagnostics. mImpl->macroExpander.lex(token);
case Token::PP_HASH: switch (token->type)
assert(false); {
break; // We should not be returning internal preprocessing tokens.
case Token::CONST_INT: // Convert preprocessing tokens to compiler tokens or report
{ // diagnostics.
int val = 0; case Token::PP_HASH:
if (!token->iValue(&val)) assert(false);
{ break;
// Do not mark the token as invalid. case Token::CONST_INT:
// Just emit the diagnostic and reset value to 0. {
mImpl->diagnostics->report(Diagnostics::INTEGER_OVERFLOW, int val = 0;
token->location, token->text); if (!token->iValue(&val))
token->text.assign("0"); {
} // Do not mark the token as invalid.
validToken = true; // Just emit the diagnostic and reset value to 0.
break; mImpl->diagnostics->report(Diagnostics::INTEGER_OVERFLOW,
} token->location, token->text);
case Token::CONST_FLOAT: token->text.assign("0");
{ }
float val = 0; validToken = true;
if (!token->fValue(&val)) break;
{ }
// Do not mark the token as invalid. case Token::CONST_FLOAT:
// Just emit the diagnostic and reset value to 0.0. {
mImpl->diagnostics->report(Diagnostics::FLOAT_OVERFLOW, float val = 0;
token->location, token->text); if (!token->fValue(&val))
token->text.assign("0.0"); {
} // Do not mark the token as invalid.
validToken = true; // Just emit the diagnostic and reset value to 0.0.
break; mImpl->diagnostics->report(Diagnostics::FLOAT_OVERFLOW,
} token->location, token->text);
case Token::PP_NUMBER: token->text.assign("0.0");
mImpl->diagnostics->report(Diagnostics::INVALID_NUMBER, }
token->location, token->text); validToken = true;
break; break;
case Token::PP_OTHER: }
mImpl->diagnostics->report(Diagnostics::INVALID_CHARACTER, case Token::PP_NUMBER:
token->location, token->text); mImpl->diagnostics->report(Diagnostics::INVALID_NUMBER,
break; token->location, token->text);
default: break;
validToken = true; case Token::PP_OTHER:
break; mImpl->diagnostics->report(Diagnostics::INVALID_CHARACTER,
} token->location, token->text);
} break;
} default:
validToken = true;
} // namespace pp break;
}
}
}
} // namespace pp
src/compiler/preprocessor/Preprocessor.h
// //
// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved. // Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. // found in the LICENSE file.
// //
#ifndef COMPILER_PREPROCESSOR_PREPROCESSOR_H_ #ifndef COMPILER_PREPROCESSOR_PREPROCESSOR_H_
#define COMPILER_PREPROCESSOR_PREPROCESSOR_H_ #define COMPILER_PREPROCESSOR_PREPROCESSOR_H_
#include <stddef.h> #include <stddef.h>
#include "pp_utils.h" #include "pp_utils.h"
namespace pp namespace pp
{ {
class Diagnostics; class Diagnostics;
class DirectiveHandler; class DirectiveHandler;
struct PreprocessorImpl; struct PreprocessorImpl;
struct Token; struct Token;
class Preprocessor class Preprocessor
{ {
public: public:
Preprocessor(Diagnostics* diagnostics, DirectiveHandler* directiveHandler); Preprocessor(Diagnostics* diagnostics, DirectiveHandler* directiveHandler);
~Preprocessor(); ~Preprocessor();
// count: specifies the number of elements in the string and length arrays. // count: specifies the number of elements in the string and length arrays.
// string: specifies an array of pointers to strings. // string: specifies an array of pointers to strings.
// length: specifies an array of string lengths. // length: specifies an array of string lengths.
// If length is NULL, each string is assumed to be null terminated. // If length is NULL, each string is assumed to be null terminated.
// If length is a value other than NULL, it points to an array containing // If length is a value other than NULL, it points to an array containing
// a string length for each of the corresponding elements of string. // a string length for each of the corresponding elements of string.
// Each element in the length array may contain the length of the // Each element in the length array may contain the length of the
// corresponding string or a value less than 0 to indicate that the string // corresponding string or a value less than 0 to indicate that the string
// is null terminated. // is null terminated.
bool init(size_t count, const char* const string[], const int length[]); bool init(size_t count, const char* const string[], const int length[]);
// Adds a pre-defined macro. // Adds a pre-defined macro.
void predefineMacro(const char* name, int value); void predefineMacro(const char* name, int value);
// Sets maximum allowed token length.
void lex(Token* token); // If token length exceeds this limit,
// the token text will be truncated to the given maximum length, and
private: // TOKEN_TOO_LONG diagnostic will be generated.
PP_DISALLOW_COPY_AND_ASSIGN(Preprocessor); // The maximum length defaults to 256.
void setMaxTokenLength(size_t maxLength);
PreprocessorImpl* mImpl;
}; void lex(Token* token);
} // namespace pp private:
#endif // COMPILER_PREPROCESSOR_PREPROCESSOR_H_ PP_DISALLOW_COPY_AND_ASSIGN(Preprocessor);
PreprocessorImpl* mImpl;
};
} // namespace pp
#endif // COMPILER_PREPROCESSOR_PREPROCESSOR_H_
src/compiler/preprocessor/Tokenizer.cpp
...@@ -2290,11 +2290,9 @@ void ppfree (void * ptr , yyscan_t yyscanner) ...@@ -2290,11 +2290,9 @@ void ppfree (void * ptr , yyscan_t yyscanner)
namespace pp { namespace pp {
// TODO(alokp): Maximum token length should ideally be specified by Tokenizer::Tokenizer(Diagnostics* diagnostics)
// the preprocessor client, i.e., the compiler. : mHandle(0),
const size_t Tokenizer::kMaxTokenLength = 256; mMaxTokenLength(256)
Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
{ {
mContext.diagnostics = diagnostics; mContext.diagnostics = diagnostics;
} }
...@@ -2327,11 +2325,11 @@ void Tokenizer::setLineNumber(int line) ...@@ -2327,11 +2325,11 @@ void Tokenizer::setLineNumber(int line)
void Tokenizer::lex(Token* token) void Tokenizer::lex(Token* token)
{ {
token->type = pplex(&token->text,&token->location,mHandle); token->type = pplex(&token->text,&token->location,mHandle);
if (token->text.size() > kMaxTokenLength) if (token->text.size() > mMaxTokenLength)
{ {
mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG, mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG,
token->location, token->text); token->location, token->text);
token->text.erase(kMaxTokenLength); token->text.erase(mMaxTokenLength);
} }
token->flags = 0; token->flags = 0;
......
src/compiler/preprocessor/Tokenizer.h
// //
// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved. // Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. // found in the LICENSE file.
// //
#ifndef COMPILER_PREPROCESSOR_TOKENIZER_H_ #ifndef COMPILER_PREPROCESSOR_TOKENIZER_H_
#define COMPILER_PREPROCESSOR_TOKENIZER_H_ #define COMPILER_PREPROCESSOR_TOKENIZER_H_
#include "Input.h" #include "Input.h"
#include "Lexer.h" #include "Lexer.h"
#include "pp_utils.h" #include "pp_utils.h"
namespace pp namespace pp
{ {
class Diagnostics; class Diagnostics;
class Tokenizer : public Lexer class Tokenizer : public Lexer
{ {
public: public:
struct Context struct Context
{ {
Diagnostics* diagnostics; Diagnostics* diagnostics;
Input input; Input input;
// The location where yytext points to. Token location should track // The location where yytext points to. Token location should track
// scanLoc instead of Input::mReadLoc because they may not be the same // scanLoc instead of Input::mReadLoc because they may not be the same
// if text is buffered up in the scanner input buffer. // if text is buffered up in the scanner input buffer.
Input::Location scanLoc; Input::Location scanLoc;
bool leadingSpace; bool leadingSpace;
bool lineStart; bool lineStart;
}; };
static const std::size_t kMaxTokenLength;
Tokenizer(Diagnostics* diagnostics);
Tokenizer(Diagnostics* diagnostics); ~Tokenizer();
~Tokenizer();
bool init(size_t count, const char* const string[], const int length[]);
bool init(size_t count, const char* const string[], const int length[]);
void setMaxTokenLength(size_t maxLength) { mMaxTokenLength = maxLength; }
void setFileNumber(int file); void setFileNumber(int file);
void setLineNumber(int line); void setLineNumber(int line);
virtual void lex(Token* token); virtual void lex(Token* token);
private: private:
PP_DISALLOW_COPY_AND_ASSIGN(Tokenizer); PP_DISALLOW_COPY_AND_ASSIGN(Tokenizer);
bool initScanner(); bool initScanner();
void destroyScanner(); void destroyScanner();
void* mHandle; // Scanner handle. void* mHandle; // Scanner handle.
Context mContext; // Scanner extra. Context mContext; // Scanner extra.
}; size_t mMaxTokenLength;
};
} // namespace pp
#endif // COMPILER_PREPROCESSOR_TOKENIZER_H_ } // namespace pp
#endif // COMPILER_PREPROCESSOR_TOKENIZER_H_
src/compiler/preprocessor/Tokenizer.l
/* /*
// //
// Copyright (c) 2002-2013 The ANGLE Project Authors. All rights reserved. // Copyright (c) 2002-2013 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. // found in the LICENSE file.
// //
This file contains the Lex specification for GLSL ES preprocessor. This file contains the Lex specification for GLSL ES preprocessor.
Based on Microsoft Visual Studio 2010 Preprocessor Grammar: Based on Microsoft Visual Studio 2010 Preprocessor Grammar:
http://msdn.microsoft.com/en-us/library/2scxys89.aspx http://msdn.microsoft.com/en-us/library/2scxys89.aspx
IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh. IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
*/ */
%top{ %top{
// //
// Copyright (c) 2011-2013 The ANGLE Project Authors. All rights reserved. // Copyright (c) 2011-2013 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. // found in the LICENSE file.
// //
// This file is auto-generated by generate_parser.sh. DO NOT EDIT! // This file is auto-generated by generate_parser.sh. DO NOT EDIT!
} }
%{ %{
#include "Tokenizer.h" #include "Tokenizer.h"
#include "DiagnosticsBase.h" #include "DiagnosticsBase.h"
#include "Token.h" #include "Token.h"
#if defined(__GNUC__) #if defined(__GNUC__)
// Triggered by the auto-generated yy_fatal_error function. // Triggered by the auto-generated yy_fatal_error function.
#pragma GCC diagnostic ignored "-Wmissing-noreturn" #pragma GCC diagnostic ignored "-Wmissing-noreturn"
#endif #endif
typedef std::string YYSTYPE; typedef std::string YYSTYPE;
typedef pp::SourceLocation YYLTYPE; typedef pp::SourceLocation YYLTYPE;
// Use the unused yycolumn variable to track file (string) number. // Use the unused yycolumn variable to track file (string) number.
#define yyfileno yycolumn #define yyfileno yycolumn
#define YY_USER_INIT \ #define YY_USER_INIT \
do { \ do { \
yyfileno = 0; \ yyfileno = 0; \
yylineno = 1; \ yylineno = 1; \
yyextra->leadingSpace = false; \ yyextra->leadingSpace = false; \
yyextra->lineStart = true; \ yyextra->lineStart = true; \
} while(0); } while(0);
#define YY_USER_ACTION \ #define YY_USER_ACTION \
do \ do \
{ \ { \
pp::Input* input = &yyextra->input; \ pp::Input* input = &yyextra->input; \
pp::Input::Location* scanLoc = &yyextra->scanLoc; \ pp::Input::Location* scanLoc = &yyextra->scanLoc; \
while ((scanLoc->sIndex < input->count()) && \ while ((scanLoc->sIndex < input->count()) && \
(scanLoc->cIndex >= input->length(scanLoc->sIndex))) \ (scanLoc->cIndex >= input->length(scanLoc->sIndex))) \
{ \ { \
scanLoc->cIndex -= input->length(scanLoc->sIndex++); \ scanLoc->cIndex -= input->length(scanLoc->sIndex++); \
++yyfileno; yylineno = 1; \ ++yyfileno; yylineno = 1; \
} \ } \
yylloc->file = yyfileno; \ yylloc->file = yyfileno; \
yylloc->line = yylineno; \ yylloc->line = yylineno; \
scanLoc->cIndex += yyleng; \ scanLoc->cIndex += yyleng; \
} while(0); } while(0);
#define YY_INPUT(buf, result, maxSize) \ #define YY_INPUT(buf, result, maxSize) \
result = yyextra->input.read(buf, maxSize); result = yyextra->input.read(buf, maxSize);
%} %}
%option noyywrap nounput never-interactive %option noyywrap nounput never-interactive
%option reentrant bison-bridge bison-locations %option reentrant bison-bridge bison-locations
%option prefix="pp" %option prefix="pp"
%option extra-type="pp::Tokenizer::Context*" %option extra-type="pp::Tokenizer::Context*"
%x COMMENT %x COMMENT
NEWLINE \n|\r|\r\n NEWLINE \n|\r|\r\n
IDENTIFIER [_a-zA-Z][_a-zA-Z0-9]* IDENTIFIER [_a-zA-Z][_a-zA-Z0-9]*
PUNCTUATOR [][<>(){}.+-/*%^|&~=!:;,?] PUNCTUATOR [][<>(){}.+-/*%^|&~=!:;,?]
DECIMAL_CONSTANT [1-9][0-9]* DECIMAL_CONSTANT [1-9][0-9]*
OCTAL_CONSTANT 0[0-7]* OCTAL_CONSTANT 0[0-7]*
HEXADECIMAL_CONSTANT 0[xX][0-9a-fA-F]+ HEXADECIMAL_CONSTANT 0[xX][0-9a-fA-F]+
DIGIT [0-9] DIGIT [0-9]
EXPONENT_PART [eE][+-]?{DIGIT}+ EXPONENT_PART [eE][+-]?{DIGIT}+
FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".") FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
%% %%
/* Line comment */ /* Line comment */
"//"[^\r\n]* "//"[^\r\n]*
/* Block comment */ /* Block comment */
/* Line breaks are just counted - not returned. */ /* Line breaks are just counted - not returned. */
/* The comment is replaced by a single space. */ /* The comment is replaced by a single space. */
"/*" { BEGIN(COMMENT); } "/*" { BEGIN(COMMENT); }
<COMMENT>[^*\r\n]+ <COMMENT>[^*\r\n]+
<COMMENT>"*" <COMMENT>"*"
<COMMENT>{NEWLINE} { ++yylineno; } <COMMENT>{NEWLINE} { ++yylineno; }
<COMMENT>"*/" { <COMMENT>"*/" {
yyextra->leadingSpace = true; yyextra->leadingSpace = true;
BEGIN(INITIAL); BEGIN(INITIAL);
} }
# { # {
// # is only valid at start of line for preprocessor directives. // # is only valid at start of line for preprocessor directives.
yylval->assign(1, yytext[0]); yylval->assign(1, yytext[0]);
return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER; return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER;
} }
{IDENTIFIER} { {IDENTIFIER} {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::IDENTIFIER; return pp::Token::IDENTIFIER;
} }
{DECIMAL_CONSTANT}|{OCTAL_CONSTANT}|{HEXADECIMAL_CONSTANT} { {DECIMAL_CONSTANT}|{OCTAL_CONSTANT}|{HEXADECIMAL_CONSTANT} {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::CONST_INT; return pp::Token::CONST_INT;
} }
({DIGIT}+{EXPONENT_PART})|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?) { ({DIGIT}+{EXPONENT_PART})|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?) {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::CONST_FLOAT; return pp::Token::CONST_FLOAT;
} }
/* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */ /* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */
/* Rule to catch all invalid integers and floats. */ /* Rule to catch all invalid integers and floats. */
({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) { ({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::PP_NUMBER; return pp::Token::PP_NUMBER;
} }
"++" { "++" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_INC; return pp::Token::OP_INC;
} }
"--" { "--" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_DEC; return pp::Token::OP_DEC;
} }
"<<" { "<<" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_LEFT; return pp::Token::OP_LEFT;
} }
">>" { ">>" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_RIGHT; return pp::Token::OP_RIGHT;
} }
"<=" { "<=" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_LE; return pp::Token::OP_LE;
} }
">=" { ">=" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_GE; return pp::Token::OP_GE;
} }
"==" { "==" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_EQ; return pp::Token::OP_EQ;
} }
"!=" { "!=" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_NE; return pp::Token::OP_NE;
} }
"&&" { "&&" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_AND; return pp::Token::OP_AND;
} }
"^^" { "^^" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_XOR; return pp::Token::OP_XOR;
} }
"||" { "||" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_OR; return pp::Token::OP_OR;
} }
"+=" { "+=" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_ADD_ASSIGN; return pp::Token::OP_ADD_ASSIGN;
} }
"-=" { "-=" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_SUB_ASSIGN; return pp::Token::OP_SUB_ASSIGN;
} }
"*=" { "*=" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_MUL_ASSIGN; return pp::Token::OP_MUL_ASSIGN;
} }
"/=" { "/=" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_DIV_ASSIGN; return pp::Token::OP_DIV_ASSIGN;
} }
"%=" { "%=" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_MOD_ASSIGN; return pp::Token::OP_MOD_ASSIGN;
} }
"<<=" { "<<=" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_LEFT_ASSIGN; return pp::Token::OP_LEFT_ASSIGN;
} }
">>=" { ">>=" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_RIGHT_ASSIGN; return pp::Token::OP_RIGHT_ASSIGN;
} }
"&=" { "&=" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_AND_ASSIGN; return pp::Token::OP_AND_ASSIGN;
} }
"^=" { "^=" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_XOR_ASSIGN; return pp::Token::OP_XOR_ASSIGN;
} }
"|=" { "|=" {
yylval->assign(yytext, yyleng); yylval->assign(yytext, yyleng);
return pp::Token::OP_OR_ASSIGN; return pp::Token::OP_OR_ASSIGN;
} }
{PUNCTUATOR} { {PUNCTUATOR} {
yylval->assign(1, yytext[0]); yylval->assign(1, yytext[0]);
return yytext[0]; return yytext[0];
} }
[ \t\v\f]+ { yyextra->leadingSpace = true; } [ \t\v\f]+ { yyextra->leadingSpace = true; }
{NEWLINE} { {NEWLINE} {
++yylineno; ++yylineno;
yylval->assign(1, '\n'); yylval->assign(1, '\n');
return '\n'; return '\n';
} }
\\{NEWLINE} { ++yylineno; } \\{NEWLINE} { ++yylineno; }
. { . {
yylval->assign(1, yytext[0]); yylval->assign(1, yytext[0]);
return pp::Token::PP_OTHER; return pp::Token::PP_OTHER;
} }
<*><<EOF>> { <*><<EOF>> {
// YY_USER_ACTION is not invoked for handling EOF. // YY_USER_ACTION is not invoked for handling EOF.
// Set the location for EOF token manually. // Set the location for EOF token manually.
pp::Input* input = &yyextra->input; pp::Input* input = &yyextra->input;
pp::Input::Location* scanLoc = &yyextra->scanLoc; pp::Input::Location* scanLoc = &yyextra->scanLoc;
yy_size_t sIndexMax = input->count() ? input->count() - 1 : 0; yy_size_t sIndexMax = input->count() ? input->count() - 1 : 0;
if (scanLoc->sIndex != sIndexMax) if (scanLoc->sIndex != sIndexMax)
{ {
// We can only reach here if there are empty strings at the // We can only reach here if there are empty strings at the
// end of the input. // end of the input.
scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0; scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0;
// FIXME: this is not 64-bit clean. // FIXME: this is not 64-bit clean.
yyfileno = static_cast<int>(sIndexMax); yylineno = 1; yyfileno = static_cast<int>(sIndexMax); yylineno = 1;
} }
yylloc->file = yyfileno; yylloc->file = yyfileno;
yylloc->line = yylineno; yylloc->line = yylineno;
yylval->clear(); yylval->clear();
if (YY_START == COMMENT) if (YY_START == COMMENT)
{ {
yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT, yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
pp::SourceLocation(yyfileno, yylineno), pp::SourceLocation(yyfileno, yylineno),
""); "");
} }
yyterminate(); yyterminate();
} }
%% %%
namespace pp { namespace pp {
// TODO(alokp): Maximum token length should ideally be specified by Tokenizer::Tokenizer(Diagnostics* diagnostics)
// the preprocessor client, i.e., the compiler. : mHandle(0),
const size_t Tokenizer::kMaxTokenLength = 256; mMaxTokenLength(256)
{
Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0) mContext.diagnostics = diagnostics;
{ }
mContext.diagnostics = diagnostics;
} Tokenizer::~Tokenizer()
{
Tokenizer::~Tokenizer() destroyScanner();
{ }
destroyScanner();
} bool Tokenizer::init(size_t count, const char* const string[], const int length[])
{
bool Tokenizer::init(size_t count, const char* const string[], const int length[]) if ((count > 0) && (string == 0)) return false;
{
if ((count > 0) && (string == 0)) return false; mContext.input = Input(count, string, length);
return initScanner();
mContext.input = Input(count, string, length); }
return initScanner();
} void Tokenizer::setFileNumber(int file)
{
void Tokenizer::setFileNumber(int file) // We use column number as file number.
{ // See macro yyfileno.
// We use column number as file number. yyset_column(file, mHandle);
// See macro yyfileno. }
yyset_column(file, mHandle);
} void Tokenizer::setLineNumber(int line)
{
void Tokenizer::setLineNumber(int line) yyset_lineno(line, mHandle);
{ }
yyset_lineno(line, mHandle);
} void Tokenizer::lex(Token* token)
{
void Tokenizer::lex(Token* token) token->type = yylex(&token->text, &token->location, mHandle);
{ if (token->text.size() > mMaxTokenLength)
token->type = yylex(&token->text, &token->location, mHandle); {
if (token->text.size() > kMaxTokenLength) mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG,
{ token->location, token->text);
mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG, token->text.erase(mMaxTokenLength);
token->location, token->text); }
token->text.erase(kMaxTokenLength);
} token->flags = 0;
token->flags = 0; token->setAtStartOfLine(mContext.lineStart);
mContext.lineStart = token->type == '\n';
token->setAtStartOfLine(mContext.lineStart);
mContext.lineStart = token->type == '\n'; token->setHasLeadingSpace(mContext.leadingSpace);
mContext.leadingSpace = false;
token->setHasLeadingSpace(mContext.leadingSpace); }
mContext.leadingSpace = false;
} bool Tokenizer::initScanner()
{
bool Tokenizer::initScanner() if ((mHandle == NULL) && yylex_init_extra(&mContext, &mHandle))
{ return false;
if ((mHandle == NULL) && yylex_init_extra(&mContext, &mHandle))
return false; yyrestart(0, mHandle);
return true;
yyrestart(0, mHandle); }
return true;
} void Tokenizer::destroyScanner()
{
void Tokenizer::destroyScanner() if (mHandle == NULL)
{ return;
if (mHandle == NULL)
return; yylex_destroy(mHandle);
mHandle = NULL;
yylex_destroy(mHandle); }
mHandle = NULL;
} } // namespace pp
} // namespace pp
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment