Skip to content

A
angle
Commit ba2cfe7c by Alok Priyadarshi
Options

Moved the definition of max-token length from preprocessor to compiler.

R=kbr@chromium.org Review URL: https://codereview.appspot.com/12957043
parent 76985f3c
Showing with 634 additions and 621 deletions
src/compiler/glslang.l
......@@ -321,6 +321,7 @@ int glslang_scan(size_t count, const char* const string[], const int length[],
// Initialize preprocessor.
if (!context->preprocessor.init(count, string, length))
return 1;
context->preprocessor.setMaxTokenLength(SH_MAX_TOKEN_LENGTH);
// Define extension macros.
const TExtensionBehavior& extBehavior = context->extensionBehavior();
......
src/compiler/glslang.y
......@@ -51,6 +51,7 @@ WHICH GENERATES THE GLSL ES PARSER (glslang_tab.cpp AND glslang_tab.h).
%code requires {
#define YYLTYPE TSourceLoc
#define YYLTYPE_IS_DECLARED 1
#define SH_MAX_TOKEN_LENGTH 256 // WebGL spec.
}
%union {
......
src/compiler/glslang_lex.cpp
......@@ -2930,6 +2930,7 @@ int glslang_scan(size_t count, const char* const string[], const int length[],
// Initialize preprocessor.
if (!context->preprocessor.init(count, string, length))
return 1;
context->preprocessor.setMaxTokenLength(SH_MAX_TOKEN_LENGTH);
// Define extension macros.
const TExtensionBehavior& extBehavior = context->extensionBehavior();
......
src/compiler/glslang_tab.cpp
......@@ -126,6 +126,7 @@ extern int yydebug;
#define YYLTYPE TSourceLoc
#define YYLTYPE_IS_DECLARED 1
#define SH_MAX_TOKEN_LENGTH 256 // WebGL spec.
......@@ -713,27 +714,27 @@ static const yytype_int16 yyrhs[] =
/* YYRLINE[YYN] -- source line where rule number YYN was defined. */
static const yytype_uint16 yyrline[] =
{
0, 179, 179, 180, 183, 226, 229, 242, 247, 252,
258, 261, 264, 267, 362, 372, 385, 393, 493, 496,
504, 507, 513, 517, 524, 530, 539, 547, 602, 612,
615, 625, 635, 656, 657, 658, 663, 664, 672, 683,
684, 692, 703, 707, 708, 718, 728, 738, 751, 752,
762, 775, 779, 783, 787, 788, 801, 802, 815, 816,
829, 830, 847, 848, 861, 862, 863, 864, 865, 869,
872, 883, 891, 918, 923, 937, 992, 995, 1002, 1010,
1031, 1052, 1062, 1090, 1095, 1105, 1110, 1120, 1123, 1126,
1129, 1135, 1142, 1145, 1167, 1185, 1209, 1232, 1236, 1254,
1262, 1294, 1314, 1335, 1344, 1367, 1370, 1376, 1384, 1392,
1400, 1410, 1417, 1420, 1423, 1429, 1432, 1447, 1451, 1455,
1459, 1463, 1468, 1473, 1478, 1483, 1488, 1493, 1498, 1503,
1508, 1513, 1518, 1523, 1527, 1531, 1539, 1547, 1551, 1564,
1564, 1578, 1578, 1587, 1590, 1606, 1639, 1643, 1649, 1656,
1671, 1675, 1679, 1680, 1686, 1687, 1688, 1689, 1690, 1694,
1695, 1695, 1695, 1705, 1706, 1710, 1710, 1711, 1711, 1716,
1719, 1729, 1732, 1738, 1739, 1743, 1751, 1755, 1765, 1770,
1787, 1787, 1792, 1792, 1799, 1799, 1807, 1810, 1816, 1819,
1825, 1829, 1836, 1843, 1850, 1857, 1868, 1877, 1881, 1888,
1891, 1897, 1897
0, 180, 180, 181, 184, 227, 230, 243, 248, 253,
259, 262, 265, 268, 363, 373, 386, 394, 494, 497,
505, 508, 514, 518, 525, 531, 540, 548, 603, 613,
616, 626, 636, 657, 658, 659, 664, 665, 673, 684,
685, 693, 704, 708, 709, 719, 729, 739, 752, 753,
763, 776, 780, 784, 788, 789, 802, 803, 816, 817,
830, 831, 848, 849, 862, 863, 864, 865, 866, 870,
873, 884, 892, 919, 924, 938, 993, 996, 1003, 1011,
1032, 1053, 1063, 1091, 1096, 1106, 1111, 1121, 1124, 1127,
1130, 1136, 1143, 1146, 1168, 1186, 1210, 1233, 1237, 1255,
1263, 1295, 1315, 1336, 1345, 1368, 1371, 1377, 1385, 1393,
1401, 1411, 1418, 1421, 1424, 1430, 1433, 1448, 1452, 1456,
1460, 1464, 1469, 1474, 1479, 1484, 1489, 1494, 1499, 1504,
1509, 1514, 1519, 1524, 1528, 1532, 1540, 1548, 1552, 1565,
1565, 1579, 1579, 1588, 1591, 1607, 1640, 1644, 1650, 1657,
1672, 1676, 1680, 1681, 1687, 1688, 1689, 1690, 1691, 1695,
1696, 1696, 1696, 1706, 1707, 1711, 1711, 1712, 1712, 1717,
1720, 1730, 1733, 1739, 1740, 1744, 1752, 1756, 1766, 1771,
1788, 1788, 1793, 1793, 1800, 1800, 1808, 1811, 1817, 1820,
1826, 1830, 1837, 1844, 1851, 1858, 1869, 1878, 1882, 1889,
1892, 1898, 1898
};
#endif
......
src/compiler/glslang_tab.h
......@@ -44,6 +44,7 @@ extern int yydebug;
#define YYLTYPE TSourceLoc
#define YYLTYPE_IS_DECLARED 1
#define SH_MAX_TOKEN_LENGTH 256 // WebGL spec.
......
src/compiler/preprocessor/Preprocessor.cpp
//
// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
#include "Preprocessor.h"
#include <cassert>
#include <sstream>
#include "DiagnosticsBase.h"
#include "DirectiveParser.h"
#include "Macro.h"
#include "MacroExpander.h"
#include "Token.h"
#include "Tokenizer.h"
namespace pp
{
struct PreprocessorImpl
{
Diagnostics* diagnostics;
MacroSet macroSet;
Tokenizer tokenizer;
DirectiveParser directiveParser;
MacroExpander macroExpander;
PreprocessorImpl(Diagnostics* diag,
DirectiveHandler* directiveHandler) :
diagnostics(diag),
tokenizer(diag),
directiveParser(&tokenizer, &macroSet, diag, directiveHandler),
macroExpander(&directiveParser, &macroSet, diag)
{
}
};
Preprocessor::Preprocessor(Diagnostics* diagnostics,
DirectiveHandler* directiveHandler)
{
mImpl = new PreprocessorImpl(diagnostics, directiveHandler);
}
Preprocessor::~Preprocessor()
{
delete mImpl;
}
bool Preprocessor::init(size_t count,
const char* const string[],
const int length[])
{
static const int kGLSLVersion = 100;
// Add standard pre-defined macros.
predefineMacro("__LINE__", 0);
predefineMacro("__FILE__", 0);
predefineMacro("__VERSION__", kGLSLVersion);
predefineMacro("GL_ES", 1);
return mImpl->tokenizer.init(count, string, length);
}
void Preprocessor::predefineMacro(const char* name, int value)
{
std::ostringstream stream;
stream << value;
Token token;
token.type = Token::CONST_INT;
token.text = stream.str();
Macro macro;
macro.predefined = true;
macro.type = Macro::kTypeObj;
macro.name = name;
macro.replacements.push_back(token);
mImpl->macroSet[name] = macro;
}
void Preprocessor::lex(Token* token)
{
bool validToken = false;
while (!validToken)
{
mImpl->macroExpander.lex(token);
switch (token->type)
{
// We should not be returning internal preprocessing tokens.
// Convert preprocessing tokens to compiler tokens or report
// diagnostics.
case Token::PP_HASH:
assert(false);
break;
case Token::CONST_INT:
{
int val = 0;
if (!token->iValue(&val))
{
// Do not mark the token as invalid.
// Just emit the diagnostic and reset value to 0.
mImpl->diagnostics->report(Diagnostics::INTEGER_OVERFLOW,
token->location, token->text);
token->text.assign("0");
}
validToken = true;
break;
}
case Token::CONST_FLOAT:
{
float val = 0;
if (!token->fValue(&val))
{
// Do not mark the token as invalid.
// Just emit the diagnostic and reset value to 0.0.
mImpl->diagnostics->report(Diagnostics::FLOAT_OVERFLOW,
token->location, token->text);
token->text.assign("0.0");
}
validToken = true;
break;
}
case Token::PP_NUMBER:
mImpl->diagnostics->report(Diagnostics::INVALID_NUMBER,
token->location, token->text);
break;
case Token::PP_OTHER:
mImpl->diagnostics->report(Diagnostics::INVALID_CHARACTER,
token->location, token->text);
break;
default:
validToken = true;
break;
}
}
}
} // namespace pp
//
// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
#include "Preprocessor.h"
#include <cassert>
#include <sstream>
#include "DiagnosticsBase.h"
#include "DirectiveParser.h"
#include "Macro.h"
#include "MacroExpander.h"
#include "Token.h"
#include "Tokenizer.h"
namespace pp
{
struct PreprocessorImpl
{
Diagnostics* diagnostics;
MacroSet macroSet;
Tokenizer tokenizer;
DirectiveParser directiveParser;
MacroExpander macroExpander;
PreprocessorImpl(Diagnostics* diag,
DirectiveHandler* directiveHandler) :
diagnostics(diag),
tokenizer(diag),
directiveParser(&tokenizer, &macroSet, diag, directiveHandler),
macroExpander(&directiveParser, &macroSet, diag)
{
}
};
Preprocessor::Preprocessor(Diagnostics* diagnostics,
DirectiveHandler* directiveHandler)
{
mImpl = new PreprocessorImpl(diagnostics, directiveHandler);
}
Preprocessor::~Preprocessor()
{
delete mImpl;
}
bool Preprocessor::init(size_t count,
const char* const string[],
const int length[])
{
static const int kGLSLVersion = 100;
// Add standard pre-defined macros.
predefineMacro("__LINE__", 0);
predefineMacro("__FILE__", 0);
predefineMacro("__VERSION__", kGLSLVersion);
predefineMacro("GL_ES", 1);
return mImpl->tokenizer.init(count, string, length);
}
void Preprocessor::predefineMacro(const char* name, int value)
{
std::ostringstream stream;
stream << value;
Token token;
token.type = Token::CONST_INT;
token.text = stream.str();
Macro macro;
macro.predefined = true;
macro.type = Macro::kTypeObj;
macro.name = name;
macro.replacements.push_back(token);
mImpl->macroSet[name] = macro;
}
void Preprocessor::setMaxTokenLength(size_t maxLength)
{
mImpl->tokenizer.setMaxTokenLength(maxLength);
}
void Preprocessor::lex(Token* token)
{
bool validToken = false;
while (!validToken)
{
mImpl->macroExpander.lex(token);
switch (token->type)
{
// We should not be returning internal preprocessing tokens.
// Convert preprocessing tokens to compiler tokens or report
// diagnostics.
case Token::PP_HASH:
assert(false);
break;
case Token::CONST_INT:
{
int val = 0;
if (!token->iValue(&val))
{
// Do not mark the token as invalid.
// Just emit the diagnostic and reset value to 0.
mImpl->diagnostics->report(Diagnostics::INTEGER_OVERFLOW,
token->location, token->text);
token->text.assign("0");
}
validToken = true;
break;
}
case Token::CONST_FLOAT:
{
float val = 0;
if (!token->fValue(&val))
{
// Do not mark the token as invalid.
// Just emit the diagnostic and reset value to 0.0.
mImpl->diagnostics->report(Diagnostics::FLOAT_OVERFLOW,
token->location, token->text);
token->text.assign("0.0");
}
validToken = true;
break;
}
case Token::PP_NUMBER:
mImpl->diagnostics->report(Diagnostics::INVALID_NUMBER,
token->location, token->text);
break;
case Token::PP_OTHER:
mImpl->diagnostics->report(Diagnostics::INVALID_CHARACTER,
token->location, token->text);
break;
default:
validToken = true;
break;
}
}
}
} // namespace pp
src/compiler/preprocessor/Preprocessor.h
//
// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
#ifndef COMPILER_PREPROCESSOR_PREPROCESSOR_H_
#define COMPILER_PREPROCESSOR_PREPROCESSOR_H_
#include <stddef.h>
#include "pp_utils.h"
namespace pp
{
class Diagnostics;
class DirectiveHandler;
struct PreprocessorImpl;
struct Token;
class Preprocessor
{
public:
Preprocessor(Diagnostics* diagnostics, DirectiveHandler* directiveHandler);
~Preprocessor();
// count: specifies the number of elements in the string and length arrays.
// string: specifies an array of pointers to strings.
// length: specifies an array of string lengths.
// If length is NULL, each string is assumed to be null terminated.
// If length is a value other than NULL, it points to an array containing
// a string length for each of the corresponding elements of string.
// Each element in the length array may contain the length of the
// corresponding string or a value less than 0 to indicate that the string
// is null terminated.
bool init(size_t count, const char* const string[], const int length[]);
// Adds a pre-defined macro.
void predefineMacro(const char* name, int value);
void lex(Token* token);
private:
PP_DISALLOW_COPY_AND_ASSIGN(Preprocessor);
PreprocessorImpl* mImpl;
};
} // namespace pp
#endif // COMPILER_PREPROCESSOR_PREPROCESSOR_H_
//
// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
#ifndef COMPILER_PREPROCESSOR_PREPROCESSOR_H_
#define COMPILER_PREPROCESSOR_PREPROCESSOR_H_
#include <stddef.h>
#include "pp_utils.h"
namespace pp
{
class Diagnostics;
class DirectiveHandler;
struct PreprocessorImpl;
struct Token;
class Preprocessor
{
public:
Preprocessor(Diagnostics* diagnostics, DirectiveHandler* directiveHandler);
~Preprocessor();
// count: specifies the number of elements in the string and length arrays.
// string: specifies an array of pointers to strings.
// length: specifies an array of string lengths.
// If length is NULL, each string is assumed to be null terminated.
// If length is a value other than NULL, it points to an array containing
// a string length for each of the corresponding elements of string.
// Each element in the length array may contain the length of the
// corresponding string or a value less than 0 to indicate that the string
// is null terminated.
bool init(size_t count, const char* const string[], const int length[]);
// Adds a pre-defined macro.
void predefineMacro(const char* name, int value);
// Sets maximum allowed token length.
// If token length exceeds this limit,
// the token text will be truncated to the given maximum length, and
// TOKEN_TOO_LONG diagnostic will be generated.
// The maximum length defaults to 256.
void setMaxTokenLength(size_t maxLength);
void lex(Token* token);
private:
PP_DISALLOW_COPY_AND_ASSIGN(Preprocessor);
PreprocessorImpl* mImpl;
};
} // namespace pp
#endif // COMPILER_PREPROCESSOR_PREPROCESSOR_H_
src/compiler/preprocessor/Tokenizer.cpp
......@@ -2290,11 +2290,9 @@ void ppfree (void * ptr , yyscan_t yyscanner)
namespace pp {
// TODO(alokp): Maximum token length should ideally be specified by
// the preprocessor client, i.e., the compiler.
const size_t Tokenizer::kMaxTokenLength = 256;
Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
Tokenizer::Tokenizer(Diagnostics* diagnostics)
: mHandle(0),
mMaxTokenLength(256)
{
mContext.diagnostics = diagnostics;
}
......@@ -2327,11 +2325,11 @@ void Tokenizer::setLineNumber(int line)
void Tokenizer::lex(Token* token)
{
token->type = pplex(&token->text,&token->location,mHandle);
if (token->text.size() > kMaxTokenLength)
if (token->text.size() > mMaxTokenLength)
{
mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG,
token->location, token->text);
token->text.erase(kMaxTokenLength);
token->text.erase(mMaxTokenLength);
}
token->flags = 0;
......
src/compiler/preprocessor/Tokenizer.h
//
// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
#ifndef COMPILER_PREPROCESSOR_TOKENIZER_H_
#define COMPILER_PREPROCESSOR_TOKENIZER_H_
#include "Input.h"
#include "Lexer.h"
#include "pp_utils.h"
namespace pp
{
class Diagnostics;
class Tokenizer : public Lexer
{
public:
struct Context
{
Diagnostics* diagnostics;
Input input;
// The location where yytext points to. Token location should track
// scanLoc instead of Input::mReadLoc because they may not be the same
// if text is buffered up in the scanner input buffer.
Input::Location scanLoc;
bool leadingSpace;
bool lineStart;
};
static const std::size_t kMaxTokenLength;
Tokenizer(Diagnostics* diagnostics);
~Tokenizer();
bool init(size_t count, const char* const string[], const int length[]);
void setFileNumber(int file);
void setLineNumber(int line);
virtual void lex(Token* token);
private:
PP_DISALLOW_COPY_AND_ASSIGN(Tokenizer);
bool initScanner();
void destroyScanner();
void* mHandle; // Scanner handle.
Context mContext; // Scanner extra.
};
} // namespace pp
#endif // COMPILER_PREPROCESSOR_TOKENIZER_H_
//
// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
#ifndef COMPILER_PREPROCESSOR_TOKENIZER_H_
#define COMPILER_PREPROCESSOR_TOKENIZER_H_
#include "Input.h"
#include "Lexer.h"
#include "pp_utils.h"
namespace pp
{
class Diagnostics;
class Tokenizer : public Lexer
{
public:
struct Context
{
Diagnostics* diagnostics;
Input input;
// The location where yytext points to. Token location should track
// scanLoc instead of Input::mReadLoc because they may not be the same
// if text is buffered up in the scanner input buffer.
Input::Location scanLoc;
bool leadingSpace;
bool lineStart;
};
Tokenizer(Diagnostics* diagnostics);
~Tokenizer();
bool init(size_t count, const char* const string[], const int length[]);
void setMaxTokenLength(size_t maxLength) { mMaxTokenLength = maxLength; }
void setFileNumber(int file);
void setLineNumber(int line);
virtual void lex(Token* token);
private:
PP_DISALLOW_COPY_AND_ASSIGN(Tokenizer);
bool initScanner();
void destroyScanner();
void* mHandle; // Scanner handle.
Context mContext; // Scanner extra.
size_t mMaxTokenLength;
};
} // namespace pp
#endif // COMPILER_PREPROCESSOR_TOKENIZER_H_
src/compiler/preprocessor/Tokenizer.l
/*
//
// Copyright (c) 2002-2013 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
This file contains the Lex specification for GLSL ES preprocessor.
Based on Microsoft Visual Studio 2010 Preprocessor Grammar:
http://msdn.microsoft.com/en-us/library/2scxys89.aspx
IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
*/
%top{
//
// Copyright (c) 2011-2013 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// This file is auto-generated by generate_parser.sh. DO NOT EDIT!
}
%{
#include "Tokenizer.h"
#include "DiagnosticsBase.h"
#include "Token.h"
#if defined(__GNUC__)
// Triggered by the auto-generated yy_fatal_error function.
#pragma GCC diagnostic ignored "-Wmissing-noreturn"
#endif
typedef std::string YYSTYPE;
typedef pp::SourceLocation YYLTYPE;
// Use the unused yycolumn variable to track file (string) number.
#define yyfileno yycolumn
#define YY_USER_INIT \
do { \
yyfileno = 0; \
yylineno = 1; \
yyextra->leadingSpace = false; \
yyextra->lineStart = true; \
} while(0);
#define YY_USER_ACTION \
do \
{ \
pp::Input* input = &yyextra->input; \
pp::Input::Location* scanLoc = &yyextra->scanLoc; \
while ((scanLoc->sIndex < input->count()) && \
(scanLoc->cIndex >= input->length(scanLoc->sIndex))) \
{ \
scanLoc->cIndex -= input->length(scanLoc->sIndex++); \
++yyfileno; yylineno = 1; \
} \
yylloc->file = yyfileno; \
yylloc->line = yylineno; \
scanLoc->cIndex += yyleng; \
} while(0);
#define YY_INPUT(buf, result, maxSize) \
result = yyextra->input.read(buf, maxSize);
%}
%option noyywrap nounput never-interactive
%option reentrant bison-bridge bison-locations
%option prefix="pp"
%option extra-type="pp::Tokenizer::Context*"
%x COMMENT
NEWLINE \n|\r|\r\n
IDENTIFIER [_a-zA-Z][_a-zA-Z0-9]*
PUNCTUATOR [][<>(){}.+-/*%^|&~=!:;,?]
DECIMAL_CONSTANT [1-9][0-9]*
OCTAL_CONSTANT 0[0-7]*
HEXADECIMAL_CONSTANT 0[xX][0-9a-fA-F]+
DIGIT [0-9]
EXPONENT_PART [eE][+-]?{DIGIT}+
FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
%%
/* Line comment */
"//"[^\r\n]*
/* Block comment */
/* Line breaks are just counted - not returned. */
/* The comment is replaced by a single space. */
"/*" { BEGIN(COMMENT); }
<COMMENT>[^*\r\n]+
<COMMENT>"*"
<COMMENT>{NEWLINE} { ++yylineno; }
<COMMENT>"*/" {
yyextra->leadingSpace = true;
BEGIN(INITIAL);
}
# {
// # is only valid at start of line for preprocessor directives.
yylval->assign(1, yytext[0]);
return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER;
}
{IDENTIFIER} {
yylval->assign(yytext, yyleng);
return pp::Token::IDENTIFIER;
}
{DECIMAL_CONSTANT}|{OCTAL_CONSTANT}|{HEXADECIMAL_CONSTANT} {
yylval->assign(yytext, yyleng);
return pp::Token::CONST_INT;
}
({DIGIT}+{EXPONENT_PART})|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?) {
yylval->assign(yytext, yyleng);
return pp::Token::CONST_FLOAT;
}
/* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */
/* Rule to catch all invalid integers and floats. */
({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) {
yylval->assign(yytext, yyleng);
return pp::Token::PP_NUMBER;
}
"++" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_INC;
}
"--" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_DEC;
}
"<<" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_LEFT;
}
">>" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_RIGHT;
}
"<=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_LE;
}
">=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_GE;
}
"==" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_EQ;
}
"!=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_NE;
}
"&&" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_AND;
}
"^^" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_XOR;
}
"||" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_OR;
}
"+=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_ADD_ASSIGN;
}
"-=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_SUB_ASSIGN;
}
"*=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_MUL_ASSIGN;
}
"/=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_DIV_ASSIGN;
}
"%=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_MOD_ASSIGN;
}
"<<=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_LEFT_ASSIGN;
}
">>=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_RIGHT_ASSIGN;
}
"&=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_AND_ASSIGN;
}
"^=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_XOR_ASSIGN;
}
"|=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_OR_ASSIGN;
}
{PUNCTUATOR} {
yylval->assign(1, yytext[0]);
return yytext[0];
}
[ \t\v\f]+ { yyextra->leadingSpace = true; }
{NEWLINE} {
++yylineno;
yylval->assign(1, '\n');
return '\n';
}
\\{NEWLINE} { ++yylineno; }
. {
yylval->assign(1, yytext[0]);
return pp::Token::PP_OTHER;
}
<*><<EOF>> {
// YY_USER_ACTION is not invoked for handling EOF.
// Set the location for EOF token manually.
pp::Input* input = &yyextra->input;
pp::Input::Location* scanLoc = &yyextra->scanLoc;
yy_size_t sIndexMax = input->count() ? input->count() - 1 : 0;
if (scanLoc->sIndex != sIndexMax)
{
// We can only reach here if there are empty strings at the
// end of the input.
scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0;
// FIXME: this is not 64-bit clean.
yyfileno = static_cast<int>(sIndexMax); yylineno = 1;
}
yylloc->file = yyfileno;
yylloc->line = yylineno;
yylval->clear();
if (YY_START == COMMENT)
{
yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
pp::SourceLocation(yyfileno, yylineno),
"");
}
yyterminate();
}
%%
namespace pp {
// TODO(alokp): Maximum token length should ideally be specified by
// the preprocessor client, i.e., the compiler.
const size_t Tokenizer::kMaxTokenLength = 256;
Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
{
mContext.diagnostics = diagnostics;
}
Tokenizer::~Tokenizer()
{
destroyScanner();
}
bool Tokenizer::init(size_t count, const char* const string[], const int length[])
{
if ((count > 0) && (string == 0)) return false;
mContext.input = Input(count, string, length);
return initScanner();
}
void Tokenizer::setFileNumber(int file)
{
// We use column number as file number.
// See macro yyfileno.
yyset_column(file, mHandle);
}
void Tokenizer::setLineNumber(int line)
{
yyset_lineno(line, mHandle);
}
void Tokenizer::lex(Token* token)
{
token->type = yylex(&token->text, &token->location, mHandle);
if (token->text.size() > kMaxTokenLength)
{
mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG,
token->location, token->text);
token->text.erase(kMaxTokenLength);
}
token->flags = 0;
token->setAtStartOfLine(mContext.lineStart);
mContext.lineStart = token->type == '\n';
token->setHasLeadingSpace(mContext.leadingSpace);
mContext.leadingSpace = false;
}
bool Tokenizer::initScanner()
{
if ((mHandle == NULL) && yylex_init_extra(&mContext, &mHandle))
return false;
yyrestart(0, mHandle);
return true;
}
void Tokenizer::destroyScanner()
{
if (mHandle == NULL)
return;
yylex_destroy(mHandle);
mHandle = NULL;
}
} // namespace pp
/*
//
// Copyright (c) 2002-2013 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
This file contains the Lex specification for GLSL ES preprocessor.
Based on Microsoft Visual Studio 2010 Preprocessor Grammar:
http://msdn.microsoft.com/en-us/library/2scxys89.aspx
IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
*/
%top{
//
// Copyright (c) 2011-2013 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// This file is auto-generated by generate_parser.sh. DO NOT EDIT!
}
%{
#include "Tokenizer.h"
#include "DiagnosticsBase.h"
#include "Token.h"
#if defined(__GNUC__)
// Triggered by the auto-generated yy_fatal_error function.
#pragma GCC diagnostic ignored "-Wmissing-noreturn"
#endif
typedef std::string YYSTYPE;
typedef pp::SourceLocation YYLTYPE;
// Use the unused yycolumn variable to track file (string) number.
#define yyfileno yycolumn
#define YY_USER_INIT \
do { \
yyfileno = 0; \
yylineno = 1; \
yyextra->leadingSpace = false; \
yyextra->lineStart = true; \
} while(0);
#define YY_USER_ACTION \
do \
{ \
pp::Input* input = &yyextra->input; \
pp::Input::Location* scanLoc = &yyextra->scanLoc; \
while ((scanLoc->sIndex < input->count()) && \
(scanLoc->cIndex >= input->length(scanLoc->sIndex))) \
{ \
scanLoc->cIndex -= input->length(scanLoc->sIndex++); \
++yyfileno; yylineno = 1; \
} \
yylloc->file = yyfileno; \
yylloc->line = yylineno; \
scanLoc->cIndex += yyleng; \
} while(0);
#define YY_INPUT(buf, result, maxSize) \
result = yyextra->input.read(buf, maxSize);
%}
%option noyywrap nounput never-interactive
%option reentrant bison-bridge bison-locations
%option prefix="pp"
%option extra-type="pp::Tokenizer::Context*"
%x COMMENT
NEWLINE \n|\r|\r\n
IDENTIFIER [_a-zA-Z][_a-zA-Z0-9]*
PUNCTUATOR [][<>(){}.+-/*%^|&~=!:;,?]
DECIMAL_CONSTANT [1-9][0-9]*
OCTAL_CONSTANT 0[0-7]*
HEXADECIMAL_CONSTANT 0[xX][0-9a-fA-F]+
DIGIT [0-9]
EXPONENT_PART [eE][+-]?{DIGIT}+
FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
%%
/* Line comment */
"//"[^\r\n]*
/* Block comment */
/* Line breaks are just counted - not returned. */
/* The comment is replaced by a single space. */
"/*" { BEGIN(COMMENT); }
<COMMENT>[^*\r\n]+
<COMMENT>"*"
<COMMENT>{NEWLINE} { ++yylineno; }
<COMMENT>"*/" {
yyextra->leadingSpace = true;
BEGIN(INITIAL);
}
# {
// # is only valid at start of line for preprocessor directives.
yylval->assign(1, yytext[0]);
return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER;
}
{IDENTIFIER} {
yylval->assign(yytext, yyleng);
return pp::Token::IDENTIFIER;
}
{DECIMAL_CONSTANT}|{OCTAL_CONSTANT}|{HEXADECIMAL_CONSTANT} {
yylval->assign(yytext, yyleng);
return pp::Token::CONST_INT;
}
({DIGIT}+{EXPONENT_PART})|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?) {
yylval->assign(yytext, yyleng);
return pp::Token::CONST_FLOAT;
}
/* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */
/* Rule to catch all invalid integers and floats. */
({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) {
yylval->assign(yytext, yyleng);
return pp::Token::PP_NUMBER;
}
"++" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_INC;
}
"--" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_DEC;
}
"<<" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_LEFT;
}
">>" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_RIGHT;
}
"<=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_LE;
}
">=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_GE;
}
"==" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_EQ;
}
"!=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_NE;
}
"&&" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_AND;
}
"^^" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_XOR;
}
"||" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_OR;
}
"+=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_ADD_ASSIGN;
}
"-=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_SUB_ASSIGN;
}
"*=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_MUL_ASSIGN;
}
"/=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_DIV_ASSIGN;
}
"%=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_MOD_ASSIGN;
}
"<<=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_LEFT_ASSIGN;
}
">>=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_RIGHT_ASSIGN;
}
"&=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_AND_ASSIGN;
}
"^=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_XOR_ASSIGN;
}
"|=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_OR_ASSIGN;
}
{PUNCTUATOR} {
yylval->assign(1, yytext[0]);
return yytext[0];
}
[ \t\v\f]+ { yyextra->leadingSpace = true; }
{NEWLINE} {
++yylineno;
yylval->assign(1, '\n');
return '\n';
}
\\{NEWLINE} { ++yylineno; }
. {
yylval->assign(1, yytext[0]);
return pp::Token::PP_OTHER;
}
<*><<EOF>> {
// YY_USER_ACTION is not invoked for handling EOF.
// Set the location for EOF token manually.
pp::Input* input = &yyextra->input;
pp::Input::Location* scanLoc = &yyextra->scanLoc;
yy_size_t sIndexMax = input->count() ? input->count() - 1 : 0;
if (scanLoc->sIndex != sIndexMax)
{
// We can only reach here if there are empty strings at the
// end of the input.
scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0;
// FIXME: this is not 64-bit clean.
yyfileno = static_cast<int>(sIndexMax); yylineno = 1;
}
yylloc->file = yyfileno;
yylloc->line = yylineno;
yylval->clear();
if (YY_START == COMMENT)
{
yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
pp::SourceLocation(yyfileno, yylineno),
"");
}
yyterminate();
}
%%
namespace pp {
Tokenizer::Tokenizer(Diagnostics* diagnostics)
: mHandle(0),
mMaxTokenLength(256)
{
mContext.diagnostics = diagnostics;
}
Tokenizer::~Tokenizer()
{
destroyScanner();
}
bool Tokenizer::init(size_t count, const char* const string[], const int length[])
{
if ((count > 0) && (string == 0)) return false;
mContext.input = Input(count, string, length);
return initScanner();
}
void Tokenizer::setFileNumber(int file)
{
// We use column number as file number.
// See macro yyfileno.
yyset_column(file, mHandle);
}
void Tokenizer::setLineNumber(int line)
{
yyset_lineno(line, mHandle);
}
void Tokenizer::lex(Token* token)
{
token->type = yylex(&token->text, &token->location, mHandle);
if (token->text.size() > mMaxTokenLength)
{
mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG,
token->location, token->text);
token->text.erase(mMaxTokenLength);
}
token->flags = 0;
token->setAtStartOfLine(mContext.lineStart);
mContext.lineStart = token->type == '\n';
token->setHasLeadingSpace(mContext.leadingSpace);
mContext.leadingSpace = false;
}
bool Tokenizer::initScanner()
{
if ((mHandle == NULL) && yylex_init_extra(&mContext, &mHandle))
return false;
yyrestart(0, mHandle);
return true;
}
void Tokenizer::destroyScanner()
{
if (mHandle == NULL)
return;
yylex_destroy(mHandle);
mHandle = NULL;
}
} // namespace pp
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment