Skip to content

A
angle
Commit 3187a38e by Olli Etuaho
Options

Make preprocessor ExpressionParser only lex what it can parse

This cleans up ExpressionParser so that the lexer only consumes one extra token in case the parser finishes. The parser will also finish with YYACCEPT once it has parsed a complete expression. This will make the preprocessor to generate a more informative unexpected token error instead of a syntax error if there are extra tokens after #if. This will also enable reusing ExpressionParser for parsing expressions in line directives. The format for a line directive that specifies both line and file numbers is as follows: #if line-expression file_expression ExpressionParser will need to be run twice for each line: first to parse line-expression and then to parse file-expression. For that reason, it is essential that ExpressionParser for line-expression stops before consuming more than one token of file-expression. BUG=angleproject:989 TEST=angle_unittests, dEQP-GLES3.functional.shaders.preprocessor.* Change-Id: I0bb92f733c18891eeddbc61e7c5bebdf1003559a Reviewed-on: https://chromium-review.googlesource.com/300962Tested-by: 's avatarOlli Etuaho <oetuaho@nvidia.com> Reviewed-by: 's avatarJamie Madill <jmadill@chromium.org> Reviewed-by: 's avatarZhenyao Mo <zmo@chromium.org> Tryjob-Request: Olli Etuaho <oetuaho@nvidia.com>
parent 3edfe034
Showing with 35 additions and 54 deletions
src/compiler/preprocessor/DirectiveParser.cpp
...@@ -893,7 +893,6 @@ int DirectiveParser::parseExpressionIf(Token *token) ...@@ -893,7 +893,6 @@ int DirectiveParser::parseExpressionIf(Token *token)
ExpressionParser expressionParser(&macroExpander, mDiagnostics); ExpressionParser expressionParser(&macroExpander, mDiagnostics);
int expression = 0; int expression = 0;
macroExpander.lex(token);
expressionParser.parse(token, &expression); expressionParser.parse(token, &expression);
// Check if there are tokens after #if expression. // Check if there are tokens after #if expression.
......
src/compiler/preprocessor/ExpressionParser.cpp
...@@ -110,6 +110,7 @@ typedef __int64 YYSTYPE; ...@@ -110,6 +110,7 @@ typedef __int64 YYSTYPE;
#include <stdint.h> #include <stdint.h>
typedef intmax_t YYSTYPE; typedef intmax_t YYSTYPE;
#endif // _MSC_VER #endif // _MSC_VER
#define YYENABLE_NLS 0 #define YYENABLE_NLS 0
#define YYLTYPE_IS_TRIVIAL 1 #define YYLTYPE_IS_TRIVIAL 1
#define YYSTYPE_IS_TRIVIAL 1 #define YYSTYPE_IS_TRIVIAL 1
...@@ -495,9 +496,9 @@ static const yytype_uint8 yytranslate[] = ...@@ -495,9 +496,9 @@ static const yytype_uint8 yytranslate[] =
/* YYRLINE[YYN] -- Source line where rule number YYN was defined. */ /* YYRLINE[YYN] -- Source line where rule number YYN was defined. */
static const yytype_uint16 yyrline[] = static const yytype_uint16 yyrline[] =
{ {
0, 105, 105, 111, 112, 118, 118, 139, 139, 160, 0, 106, 106, 113, 114, 124, 124, 145, 145, 166,
163, 166, 169, 172, 175, 178, 181, 184, 187, 190, 169, 172, 175, 178, 181, 184, 187, 190, 193, 196,
193, 196, 199, 222, 245, 248, 251, 254, 257, 260 199, 202, 205, 224, 243, 246, 249, 252, 255, 258
}; };
#endif #endif
...@@ -1330,6 +1331,7 @@ yyreduce: ...@@ -1330,6 +1331,7 @@ yyreduce:
{ {
*(context->result) = static_cast<int>((yyvsp[0])); *(context->result) = static_cast<int>((yyvsp[0]));
YYACCEPT;
} }
break; break;
...@@ -1339,8 +1341,12 @@ yyreduce: ...@@ -1339,8 +1341,12 @@ yyreduce:
{ {
if (!context->isIgnoringErrors()) if (!context->isIgnoringErrors())
{ {
YYABORT; // This rule should be applied right after the token is lexed, so we can
// refer to context->token in the error message.
context->diagnostics->report(pp::Diagnostics::PP_CONDITIONAL_UNEXPECTED_TOKEN,
context->token->location, context->token->text);
} }
(yyval) = (yyvsp[0]);
} }
break; break;
...@@ -1518,11 +1524,7 @@ yyreduce: ...@@ -1518,11 +1524,7 @@ yyreduce:
{ {
if ((yyvsp[0]) == 0) if ((yyvsp[0]) == 0)
{ {
if (context->isIgnoringErrors()) if (!context->isIgnoringErrors())
{
(yyval) = static_cast<YYSTYPE>(0);
}
else
{ {
std::ostringstream stream; std::ostringstream stream;
stream << (yyvsp[-2]) << " % " << (yyvsp[0]); stream << (yyvsp[-2]) << " % " << (yyvsp[0]);
...@@ -1530,8 +1532,8 @@ yyreduce: ...@@ -1530,8 +1532,8 @@ yyreduce:
context->diagnostics->report(pp::Diagnostics::PP_DIVISION_BY_ZERO, context->diagnostics->report(pp::Diagnostics::PP_DIVISION_BY_ZERO,
context->token->location, context->token->location,
text.c_str()); text.c_str());
YYABORT;
} }
(yyval) = static_cast<YYSTYPE>(0);
} }
else else
{ {
...@@ -1546,11 +1548,7 @@ yyreduce: ...@@ -1546,11 +1548,7 @@ yyreduce:
{ {
if ((yyvsp[0]) == 0) if ((yyvsp[0]) == 0)
{ {
if (context->isIgnoringErrors()) if (!context->isIgnoringErrors())
{
(yyval) = static_cast<YYSTYPE>(0);
}
else
{ {
std::ostringstream stream; std::ostringstream stream;
stream << (yyvsp[-2]) << " / " << (yyvsp[0]); stream << (yyvsp[-2]) << " / " << (yyvsp[0]);
...@@ -1558,8 +1556,8 @@ yyreduce: ...@@ -1558,8 +1556,8 @@ yyreduce:
context->diagnostics->report(pp::Diagnostics::PP_DIVISION_BY_ZERO, context->diagnostics->report(pp::Diagnostics::PP_DIVISION_BY_ZERO,
context->token->location, context->token->location,
text.c_str()); text.c_str());
YYABORT;
} }
(yyval) = static_cast<YYSTYPE>(0);
} }
else else
{ {
...@@ -1851,9 +1849,11 @@ yyreturn: ...@@ -1851,9 +1849,11 @@ yyreturn:
int yylex(YYSTYPE *lvalp, Context *context) int yylex(YYSTYPE *lvalp, Context *context)
{ {
pp::Token *token = context->token;
context->lexer->lex(token);
int type = 0; int type = 0;
pp::Token *token = context->token;
switch (token->type) switch (token->type)
{ {
case pp::Token::CONST_INT: { case pp::Token::CONST_INT: {
...@@ -1868,11 +1868,6 @@ int yylex(YYSTYPE *lvalp, Context *context) ...@@ -1868,11 +1868,6 @@ int yylex(YYSTYPE *lvalp, Context *context)
break; break;
} }
case pp::Token::IDENTIFIER: case pp::Token::IDENTIFIER:
if (!context->isIgnoringErrors())
{
context->diagnostics->report(pp::Diagnostics::PP_CONDITIONAL_UNEXPECTED_TOKEN,
token->location, token->text);
}
*lvalp = static_cast<YYSTYPE>(-1); *lvalp = static_cast<YYSTYPE>(-1);
type = TOK_IDENTIFIER; type = TOK_IDENTIFIER;
break; break;
...@@ -1921,10 +1916,6 @@ int yylex(YYSTYPE *lvalp, Context *context) ...@@ -1921,10 +1916,6 @@ int yylex(YYSTYPE *lvalp, Context *context)
break; break;
} }
// Advance to the next token if the current one is valid.
if (type != 0)
context->lexer->lex(token);
return type; return type;
} }
......
src/compiler/preprocessor/ExpressionParser.y
...@@ -52,6 +52,7 @@ typedef __int64 YYSTYPE; ...@@ -52,6 +52,7 @@ typedef __int64 YYSTYPE;
#include <stdint.h> #include <stdint.h>
typedef intmax_t YYSTYPE; typedef intmax_t YYSTYPE;
#endif // _MSC_VER #endif // _MSC_VER
#define YYENABLE_NLS 0 #define YYENABLE_NLS 0
#define YYLTYPE_IS_TRIVIAL 1 #define YYLTYPE_IS_TRIVIAL 1
#define YYSTYPE_IS_TRIVIAL 1 #define YYSTYPE_IS_TRIVIAL 1
...@@ -104,6 +105,7 @@ static void yyerror(Context* context, const char* reason); ...@@ -104,6 +105,7 @@ static void yyerror(Context* context, const char* reason);
input input
: expression { : expression {
*(context->result) = static_cast<int>($1); *(context->result) = static_cast<int>($1);
YYACCEPT;
} }
; ;
...@@ -112,8 +114,12 @@ expression ...@@ -112,8 +114,12 @@ expression
| TOK_IDENTIFIER { | TOK_IDENTIFIER {
if (!context->isIgnoringErrors()) if (!context->isIgnoringErrors())
{ {
YYABORT; // This rule should be applied right after the token is lexed, so we can
// refer to context->token in the error message.
context->diagnostics->report(pp::Diagnostics::PP_CONDITIONAL_UNEXPECTED_TOKEN,
context->token->location, context->token->text);
} }
$$ = $1;
} }
| expression TOK_OP_OR { | expression TOK_OP_OR {
if ($1 != 0) if ($1 != 0)
...@@ -199,11 +205,7 @@ expression ...@@ -199,11 +205,7 @@ expression
| expression '%' expression { | expression '%' expression {
if ($3 == 0) if ($3 == 0)
{ {
if (context->isIgnoringErrors()) if (!context->isIgnoringErrors())
{
$$ = static_cast<YYSTYPE>(0);
}
else
{ {
std::ostringstream stream; std::ostringstream stream;
stream << $1 << " % " << $3; stream << $1 << " % " << $3;
...@@ -211,8 +213,8 @@ expression ...@@ -211,8 +213,8 @@ expression
context->diagnostics->report(pp::Diagnostics::PP_DIVISION_BY_ZERO, context->diagnostics->report(pp::Diagnostics::PP_DIVISION_BY_ZERO,
context->token->location, context->token->location,
text.c_str()); text.c_str());
YYABORT;
} }
$$ = static_cast<YYSTYPE>(0);
} }
else else
{ {
...@@ -222,11 +224,7 @@ expression ...@@ -222,11 +224,7 @@ expression
| expression '/' expression { | expression '/' expression {
if ($3 == 0) if ($3 == 0)
{ {
if (context->isIgnoringErrors()) if (!context->isIgnoringErrors())
{
$$ = static_cast<YYSTYPE>(0);
}
else
{ {
std::ostringstream stream; std::ostringstream stream;
stream << $1 << " / " << $3; stream << $1 << " / " << $3;
...@@ -234,8 +232,8 @@ expression ...@@ -234,8 +232,8 @@ expression
context->diagnostics->report(pp::Diagnostics::PP_DIVISION_BY_ZERO, context->diagnostics->report(pp::Diagnostics::PP_DIVISION_BY_ZERO,
context->token->location, context->token->location,
text.c_str()); text.c_str());
YYABORT;
} }
$$ = static_cast<YYSTYPE>(0);
} }
else else
{ {
...@@ -266,9 +264,11 @@ expression ...@@ -266,9 +264,11 @@ expression
int yylex(YYSTYPE *lvalp, Context *context) int yylex(YYSTYPE *lvalp, Context *context)
{ {
pp::Token *token = context->token;
context->lexer->lex(token);
int type = 0; int type = 0;
pp::Token *token = context->token;
switch (token->type) switch (token->type)
{ {
case pp::Token::CONST_INT: { case pp::Token::CONST_INT: {
...@@ -283,11 +283,6 @@ int yylex(YYSTYPE *lvalp, Context *context) ...@@ -283,11 +283,6 @@ int yylex(YYSTYPE *lvalp, Context *context)
break; break;
} }
case pp::Token::IDENTIFIER: case pp::Token::IDENTIFIER:
if (!context->isIgnoringErrors())
{
context->diagnostics->report(pp::Diagnostics::PP_CONDITIONAL_UNEXPECTED_TOKEN,
token->location, token->text);
}
*lvalp = static_cast<YYSTYPE>(-1); *lvalp = static_cast<YYSTYPE>(-1);
type = TOK_IDENTIFIER; type = TOK_IDENTIFIER;
break; break;
...@@ -336,10 +331,6 @@ int yylex(YYSTYPE *lvalp, Context *context) ...@@ -336,10 +331,6 @@ int yylex(YYSTYPE *lvalp, Context *context)
break; break;
} }
// Advance to the next token if the current one is valid.
if (type != 0)
context->lexer->lex(token);
return type; return type;
} }
......
src/tests/preprocessor_tests/if_test.cpp
...@@ -838,8 +838,8 @@ TEST_F(IfTest, ExtraIntExpression) ...@@ -838,8 +838,8 @@ TEST_F(IfTest, ExtraIntExpression)
"#endif\n"; "#endif\n";
ASSERT_TRUE(mPreprocessor.init(1, &str, 0)); ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::PP_INVALID_EXPRESSION, EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::PP_CONDITIONAL_UNEXPECTED_TOKEN,
pp::SourceLocation(0, 1), "syntax error")); pp::SourceLocation(0, 1), "1"));
pp::Token token; pp::Token token;
mPreprocessor.lex(&token); mPreprocessor.lex(&token);
...@@ -855,8 +855,8 @@ TEST_F(IfTest, ExtraIdentifierExpression) ...@@ -855,8 +855,8 @@ TEST_F(IfTest, ExtraIdentifierExpression)
"#endif\n"; "#endif\n";
ASSERT_TRUE(mPreprocessor.init(1, &str, 0)); ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::PP_INVALID_EXPRESSION, EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::PP_CONDITIONAL_UNEXPECTED_TOKEN,
pp::SourceLocation(0, 2), "syntax error")); pp::SourceLocation(0, 2), "1"));
pp::Token token; pp::Token token;
mPreprocessor.lex(&token); mPreprocessor.lex(&token);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment