Commit 01fd431c by alokp@chromium.org

More prep work for macro expansion. Off-loaded lexing from Context class to a new Lexer class.

Review URL: https://codereview.appspot.com/5059048 git-svn-id: https://angleproject.googlecode.com/svn/trunk@1011 736b8ea6-26fd-11df-bfd4-992fa37f6226
parent 43affc5e
...@@ -10,6 +10,7 @@ ...@@ -10,6 +10,7 @@
#include <sstream> #include <sstream>
#include "compiler/debug.h" #include "compiler/debug.h"
#include "Lexer.h"
#include "stl_utils.h" #include "stl_utils.h"
#include "token_type.h" #include "token_type.h"
...@@ -31,24 +32,12 @@ static bool isMacroNameReserved(const std::string* name) ...@@ -31,24 +32,12 @@ static bool isMacroNameReserved(const std::string* name)
namespace pp namespace pp
{ {
Context::Context() Context::Context() : mOutput(NULL)
: mLexer(NULL),
mInput(NULL),
mOutput(NULL)
{ {
} }
Context::~Context() Context::~Context()
{ {
destroyLexer();
}
bool Context::init()
{
return initLexer();
// TODO(alokp): Define built-in macros here so that we do not need to
// define them everytime in process().
} }
bool Context::process(int count, bool Context::process(int count,
...@@ -59,7 +48,8 @@ bool Context::process(int count, ...@@ -59,7 +48,8 @@ bool Context::process(int count,
ASSERT((count >=0) && (string != NULL) && (output != NULL)); ASSERT((count >=0) && (string != NULL) && (output != NULL));
// Setup. // Setup.
mInput = new Input(count, string, length); mLexer.reset(new Lexer);
if (!mLexer->init(count, string, length)) return false;
mOutput = output; mOutput = output;
defineBuiltInMacro("GL_ES", 1); defineBuiltInMacro("GL_ES", 1);
...@@ -71,6 +61,11 @@ bool Context::process(int count, ...@@ -71,6 +61,11 @@ bool Context::process(int count,
return success; return success;
} }
int Context::lex(YYSTYPE* lvalp, YYLTYPE* llocp)
{
return mLexer->lex(lvalp, llocp);
}
bool Context::defineMacro(pp::Token::Location location, bool Context::defineMacro(pp::Token::Location location,
pp::Macro::Type type, pp::Macro::Type type,
std::string* name, std::string* name,
...@@ -116,10 +111,8 @@ void Context::reset() ...@@ -116,10 +111,8 @@ void Context::reset()
std::for_each(mMacros.begin(), mMacros.end(), DeleteSecond()); std::for_each(mMacros.begin(), mMacros.end(), DeleteSecond());
mMacros.clear(); mMacros.clear();
delete mInput;
mInput = NULL;
mOutput = NULL; mOutput = NULL;
mLexer.reset();
} }
void Context::defineBuiltInMacro(const std::string& name, int value) void Context::defineBuiltInMacro(const std::string& name, int value)
......
...@@ -10,27 +10,30 @@ ...@@ -10,27 +10,30 @@
#include <map> #include <map>
#include "common/angleutils.h" #include "common/angleutils.h"
#include "Input.h"
#include "Macro.h" #include "Macro.h"
#include "Token.h" #include "Token.h"
struct YYLTYPE;
union YYSTYPE;
namespace pp namespace pp
{ {
class Lexer;
class Context class Context
{ {
public: public:
Context(); Context();
~Context(); ~Context();
bool init();
bool process(int count, const char* const string[], const int length[], bool process(int count, const char* const string[], const int length[],
TokenVector* output); TokenVector* output);
void* lexer() { return mLexer; }
int readInput(char* buf, int maxSize);
TokenVector* output() { return mOutput; } TokenVector* output() { return mOutput; }
int lex(YYSTYPE* lvalp, YYLTYPE* llocp);
bool defineMacro(pp::Token::Location location, bool defineMacro(pp::Token::Location location,
pp::Macro::Type type, pp::Macro::Type type,
std::string* name, std::string* name,
...@@ -44,13 +47,10 @@ class Context ...@@ -44,13 +47,10 @@ class Context
typedef std::map<std::string, Macro*> MacroSet; typedef std::map<std::string, Macro*> MacroSet;
void reset(); void reset();
bool initLexer();
void destroyLexer();
void defineBuiltInMacro(const std::string& name, int value); void defineBuiltInMacro(const std::string& name, int value);
bool parse(); bool parse();
void* mLexer; // Lexer handle. std::auto_ptr<Lexer> mLexer;
Input* mInput;
TokenVector* mOutput; TokenVector* mOutput;
MacroSet mMacros; // Defined macros. MacroSet mMacros; // Defined macros.
}; };
......
//
// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
#include "Lexer.h"
#include "Input.h"
namespace pp
{
Lexer::Lexer() : mHandle(0)
{
}
Lexer::~Lexer()
{
destroyLexer();
}
bool Lexer::init(int count, const char* const string[], const int length[])
{
mInput.reset(new Input(count, string, length));
return initLexer();
}
} // namespace pp
//
// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
#ifndef COMPILER_PREPROCESSOR_LEXER_H_
#define COMPILER_PREPROCESSOR_LEXER_H_
#include <memory>
#include "common/angleutils.h"
struct YYLTYPE;
union YYSTYPE;
namespace pp
{
class Input;
class Lexer
{
public:
Lexer();
~Lexer();
bool init(int count, const char* const string[], const int length[]);
int lex(YYSTYPE* lvalp, YYLTYPE* llocp);
private:
DISALLOW_COPY_AND_ASSIGN(Lexer);
bool initLexer();
void destroyLexer();
void* mHandle; // Lexer handle.
std::auto_ptr<Input> mInput; // Input buffer.
};
} // namespace pp
#endif // COMPILER_PREPROCESSOR_BUFFER_LEXER_H_
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
namespace pp namespace pp
{ {
Preprocessor::Preprocessor() : mContext(NULL) Preprocessor::Preprocessor() : mContext(new Context)
{ {
} }
...@@ -24,12 +24,6 @@ Preprocessor::~Preprocessor() ...@@ -24,12 +24,6 @@ Preprocessor::~Preprocessor()
delete mContext; delete mContext;
} }
bool Preprocessor::init()
{
mContext = new Context;
return mContext->init();
}
bool Preprocessor::process(int count, bool Preprocessor::process(int count,
const char* const string[], const char* const string[],
const int length[]) const int length[])
......
...@@ -21,8 +21,6 @@ class Preprocessor ...@@ -21,8 +21,6 @@ class Preprocessor
Preprocessor(); Preprocessor();
~Preprocessor(); ~Preprocessor();
bool init();
bool process(int count, const char* const string[], const int length[]); bool process(int count, const char* const string[], const int length[]);
TokenIterator begin() const { return mTokens.begin(); } TokenIterator begin() const { return mTokens.begin(); }
TokenIterator end() const { return mTokens.end(); } TokenIterator end() const { return mTokens.end(); }
......
...@@ -24,8 +24,10 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh. ...@@ -24,8 +24,10 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
%{ %{
#include "compiler/debug.h" #include "compiler/debug.h"
#include "Context.h" #include "Input.h"
#include "pp_tab.h" #include "Lexer.h"
#include "Token.h"
#include "token_type.h"
#define YY_USER_ACTION \ #define YY_USER_ACTION \
do { \ do { \
...@@ -35,16 +37,16 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh. ...@@ -35,16 +37,16 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
} while(0); } while(0);
#define YY_INPUT(buf, result, maxSize) \ #define YY_INPUT(buf, result, maxSize) \
result = yyextra->readInput(buf, maxSize); result = readInput(buf, maxSize, yyscanner);
static int readInput(char* buf, int maxSize, yyscan_t scanner);
static std::string* extractMacroName(const char* str, int len); static std::string* extractMacroName(const char* str, int len);
%} %}
%option noyywrap nounput never-interactive %option noyywrap nounput never-interactive
%option yylineno reentrant bison-bridge bison-locations %option yylineno reentrant bison-bridge bison-locations
%option stack
%option prefix="pp" %option prefix="pp"
%option extra-type="pp::Context*" %option extra-type="pp::Input*"
HSPACE [ \t] HSPACE [ \t]
HASH ^{HSPACE}*#{HSPACE}* HASH ^{HSPACE}*#{HSPACE}*
...@@ -115,6 +117,29 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".") ...@@ -115,6 +117,29 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
%% %%
int readInput(char* buf, int maxSize, yyscan_t scanner)
{
int nread = YY_NULL;
pp::Input* input = yyget_extra(scanner);
while (!input->eof() &&
(input->error() == pp::Input::kErrorNone) &&
(nread == YY_NULL))
{
int line = 0, file = 0;
pp::Token::decodeLocation(yyget_lineno(scanner), &line, &file);
file = input->stringIndex();
yyset_lineno(pp::Token::encodeLocation(line, file), scanner);
nread = input->read(buf, maxSize);
if (input->error() == pp::Input::kErrorUnexpectedEOF)
{
// TODO(alokp): Report error.
}
}
return nread;
}
std::string* extractMacroName(const char* str, int len) std::string* extractMacroName(const char* str, int len)
{ {
// The input string is of the form {HASH}define{HSPACE}+{IDENTIFIER} // The input string is of the form {HASH}define{HSPACE}+{IDENTIFIER}
...@@ -136,45 +161,27 @@ std::string* extractMacroName(const char* str, int len) ...@@ -136,45 +161,27 @@ std::string* extractMacroName(const char* str, int len)
namespace pp { namespace pp {
int Context::readInput(char* buf, int maxSize) int Lexer::lex(YYSTYPE* lvalp, YYLTYPE* llocp)
{ {
int nread = YY_NULL; return yylex(lvalp, llocp, mHandle);
while (!mInput->eof() &&
(mInput->error() == pp::Input::kErrorNone) &&
(nread == YY_NULL))
{
int line = 0, file = 0;
pp::Token::decodeLocation(yyget_lineno(mLexer), &line, &file);
file = mInput->stringIndex();
yyset_lineno(pp::Token::encodeLocation(line, file), mLexer);
nread = mInput->read(buf, maxSize);
if (mInput->error() == pp::Input::kErrorUnexpectedEOF)
{
// TODO(alokp): Report error.
}
}
return nread;
} }
bool Context::initLexer() bool Lexer::initLexer()
{ {
ASSERT(mLexer == NULL); if ((mHandle == NULL) && yylex_init_extra(mInput.get(), &mHandle))
if (yylex_init_extra(this, &mLexer))
return false; return false;
yyrestart(0, mLexer); yyrestart(0, mHandle);
return true; return true;
} }
void Context::destroyLexer() void Lexer::destroyLexer()
{ {
ASSERT(mLexer); if (mHandle == NULL)
return;
yylex_destroy(mLexer); yylex_destroy(mHandle);
mLexer = NULL; mHandle = NULL;
} }
} // namespace pp } // namespace pp
......
...@@ -22,9 +22,13 @@ WHICH GENERATES THE GLSL ES PARSER. ...@@ -22,9 +22,13 @@ WHICH GENERATES THE GLSL ES PARSER.
// This file is auto-generated by generate_glslang_parser.sh. DO NOT EDIT! // This file is auto-generated by generate_glslang_parser.sh. DO NOT EDIT!
#if defined(__GNUC__)
#elif defined(_MSC_VER)
#pragma warning(disable: 4065)
#endif
#include "Context.h" #include "Context.h"
#define YYLEX_PARAM context->lexer()
#define YYDEBUG 1 #define YYDEBUG 1
%} %}
...@@ -32,6 +36,7 @@ WHICH GENERATES THE GLSL ES PARSER. ...@@ -32,6 +36,7 @@ WHICH GENERATES THE GLSL ES PARSER.
%name-prefix="pp" %name-prefix="pp"
%locations %locations
%parse-param {pp::Context* context} %parse-param {pp::Context* context}
%lex-param {pp::Context* context}
%union { %union {
int ival; int ival;
...@@ -41,7 +46,7 @@ WHICH GENERATES THE GLSL ES PARSER. ...@@ -41,7 +46,7 @@ WHICH GENERATES THE GLSL ES PARSER.
} }
%{ %{
extern int yylex(YYSTYPE* lvalp, YYLTYPE* llocp, void* lexer); static int yylex(YYSTYPE* lvalp, YYLTYPE* llocp, pp::Context* context);
static void yyerror(YYLTYPE* llocp, static void yyerror(YYLTYPE* llocp,
pp::Context* context, pp::Context* context,
const char* reason); const char* reason);
...@@ -203,6 +208,11 @@ operator ...@@ -203,6 +208,11 @@ operator
%% %%
int yylex(YYSTYPE* lvalp, YYLTYPE* llocp, pp::Context* context)
{
return context->lex(lvalp, llocp);
}
void yyerror(YYLTYPE* llocp, pp::Context* context, const char* reason) void yyerror(YYLTYPE* llocp, pp::Context* context, const char* reason)
{ {
} }
......
...@@ -552,8 +552,10 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh. ...@@ -552,8 +552,10 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
*/ */
#include "compiler/debug.h" #include "compiler/debug.h"
#include "Context.h" #include "Input.h"
#include "pp_tab.h" #include "Lexer.h"
#include "Token.h"
#include "token_type.h"
#define YY_USER_ACTION \ #define YY_USER_ACTION \
do { \ do { \
...@@ -563,13 +565,14 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh. ...@@ -563,13 +565,14 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
} while(0); } while(0);
#define YY_INPUT(buf, result, maxSize) \ #define YY_INPUT(buf, result, maxSize) \
result = yyextra->readInput(buf, maxSize); result = readInput(buf, maxSize, yyscanner);
static int readInput(char* buf, int maxSize, yyscan_t scanner);
static std::string* extractMacroName(const char* str, int len); static std::string* extractMacroName(const char* str, int len);
#define INITIAL 0 #define INITIAL 0
#define YY_EXTRA_TYPE pp::Context* #define YY_EXTRA_TYPE pp::Input*
/* Holds the entire state of the reentrant scanner. */ /* Holds the entire state of the reentrant scanner. */
struct yyguts_t struct yyguts_t
...@@ -688,12 +691,6 @@ static int input (yyscan_t yyscanner ); ...@@ -688,12 +691,6 @@ static int input (yyscan_t yyscanner );
#endif #endif
static void yy_push_state (int new_state ,yyscan_t yyscanner);
static void yy_pop_state (yyscan_t yyscanner );
static int yy_top_state (yyscan_t yyscanner );
/* Amount of stuff to slurp up with each read. */ /* Amount of stuff to slurp up with each read. */
#ifndef YY_READ_BUF_SIZE #ifndef YY_READ_BUF_SIZE
#define YY_READ_BUF_SIZE 8192 #define YY_READ_BUF_SIZE 8192
...@@ -1790,46 +1787,6 @@ YY_BUFFER_STATE pp_scan_bytes (yyconst char * yybytes, int _yybytes_len , yysc ...@@ -1790,46 +1787,6 @@ YY_BUFFER_STATE pp_scan_bytes (yyconst char * yybytes, int _yybytes_len , yysc
return b; return b;
} }
static void yy_push_state (int new_state , yyscan_t yyscanner)
{
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
if ( yyg->yy_start_stack_ptr >= yyg->yy_start_stack_depth )
{
yy_size_t new_size;
yyg->yy_start_stack_depth += YY_START_STACK_INCR;
new_size = yyg->yy_start_stack_depth * sizeof( int );
if ( ! yyg->yy_start_stack )
yyg->yy_start_stack = (int *) ppalloc(new_size ,yyscanner );
else
yyg->yy_start_stack = (int *) pprealloc((void *) yyg->yy_start_stack,new_size ,yyscanner );
if ( ! yyg->yy_start_stack )
YY_FATAL_ERROR( "out of memory expanding start-condition stack" );
}
yyg->yy_start_stack[yyg->yy_start_stack_ptr++] = YY_START;
BEGIN(new_state);
}
static void yy_pop_state (yyscan_t yyscanner)
{
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
if ( --yyg->yy_start_stack_ptr < 0 )
YY_FATAL_ERROR( "start-condition stack underflow" );
BEGIN(yyg->yy_start_stack[yyg->yy_start_stack_ptr]);
}
static int yy_top_state (yyscan_t yyscanner)
{
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
return yyg->yy_start_stack[yyg->yy_start_stack_ptr - 1];
}
#ifndef YY_EXIT_FAILURE #ifndef YY_EXIT_FAILURE
#define YY_EXIT_FAILURE 2 #define YY_EXIT_FAILURE 2
#endif #endif
...@@ -2202,6 +2159,29 @@ void ppfree (void * ptr , yyscan_t yyscanner) ...@@ -2202,6 +2159,29 @@ void ppfree (void * ptr , yyscan_t yyscanner)
#define YYTABLES_NAME "yytables" #define YYTABLES_NAME "yytables"
int readInput(char* buf, int maxSize, yyscan_t scanner)
{
int nread = YY_NULL;
pp::Input* input = ppget_extra(scanner);
while (!input->eof() &&
(input->error() == pp::Input::kErrorNone) &&
(nread == YY_NULL))
{
int line = 0, file = 0;
pp::Token::decodeLocation(ppget_lineno(scanner), &line, &file);
file = input->stringIndex();
ppset_lineno(pp::Token::encodeLocation(line, file),scanner);
nread = input->read(buf, maxSize);
if (input->error() == pp::Input::kErrorUnexpectedEOF)
{
// TODO(alokp): Report error.
}
}
return nread;
}
std::string* extractMacroName(const char* str, int len) std::string* extractMacroName(const char* str, int len)
{ {
// The input string is of the form {HASH}define{HSPACE}+{IDENTIFIER} // The input string is of the form {HASH}define{HSPACE}+{IDENTIFIER}
...@@ -2223,45 +2203,27 @@ std::string* extractMacroName(const char* str, int len) ...@@ -2223,45 +2203,27 @@ std::string* extractMacroName(const char* str, int len)
namespace pp { namespace pp {
int Context::readInput(char* buf, int maxSize) int Lexer::lex(YYSTYPE* lvalp, YYLTYPE* llocp)
{ {
int nread = YY_NULL; return pplex(lvalp,llocp,mHandle);
while (!mInput->eof() &&
(mInput->error() == pp::Input::kErrorNone) &&
(nread == YY_NULL))
{
int line = 0, file = 0;
pp::Token::decodeLocation(ppget_lineno(mLexer), &line, &file);
file = mInput->stringIndex();
ppset_lineno(pp::Token::encodeLocation(line, file),mLexer);
nread = mInput->read(buf, maxSize);
if (mInput->error() == pp::Input::kErrorUnexpectedEOF)
{
// TODO(alokp): Report error.
}
}
return nread;
} }
bool Context::initLexer() bool Lexer::initLexer()
{ {
ASSERT(mLexer == NULL); if ((mHandle == NULL) && pplex_init_extra(mInput.get(),&mHandle))
if (pplex_init_extra(this,&mLexer))
return false; return false;
pprestart(0,mLexer); pprestart(0,mHandle);
return true; return true;
} }
void Context::destroyLexer() void Lexer::destroyLexer()
{ {
ASSERT(mLexer); if (mHandle == NULL)
return;
pplex_destroy(mLexer); pplex_destroy(mHandle);
mLexer = NULL; mHandle = NULL;
} }
} // namespace pp } // namespace pp
......
...@@ -130,9 +130,13 @@ ...@@ -130,9 +130,13 @@
// This file is auto-generated by generate_glslang_parser.sh. DO NOT EDIT! // This file is auto-generated by generate_glslang_parser.sh. DO NOT EDIT!
#if defined(__GNUC__)
#elif defined(_MSC_VER)
#pragma warning(disable: 4065)
#endif
#include "Context.h" #include "Context.h"
#define YYLEX_PARAM context->lexer()
#define YYDEBUG 1 #define YYDEBUG 1
...@@ -188,7 +192,7 @@ typedef struct YYLTYPE ...@@ -188,7 +192,7 @@ typedef struct YYLTYPE
/* Copy the second part of user declarations. */ /* Copy the second part of user declarations. */
extern int yylex(YYSTYPE* lvalp, YYLTYPE* llocp, void* lexer); static int yylex(YYSTYPE* lvalp, YYLTYPE* llocp, pp::Context* context);
static void yyerror(YYLTYPE* llocp, static void yyerror(YYLTYPE* llocp,
pp::Context* context, pp::Context* context,
const char* reason); const char* reason);
...@@ -504,13 +508,13 @@ static const yytype_int8 yyrhs[] = ...@@ -504,13 +508,13 @@ static const yytype_int8 yyrhs[] =
/* YYRLINE[YYN] -- source line where rule number YYN was defined. */ /* YYRLINE[YYN] -- source line where rule number YYN was defined. */
static const yytype_uint8 yyrline[] = static const yytype_uint8 yyrline[] =
{ {
0, 63, 63, 65, 69, 70, 74, 75, 84, 85, 0, 68, 68, 70, 74, 75, 79, 80, 89, 90,
88, 91, 94, 97, 100, 103, 105, 107, 110, 111, 93, 96, 99, 102, 105, 108, 110, 112, 115, 116,
112, 113, 114, 118, 119, 122, 123, 127, 133, 137, 117, 118, 119, 123, 124, 127, 128, 132, 138, 142,
144, 148, 155, 157, 159, 163, 166, 169, 172, 178, 149, 153, 160, 162, 164, 168, 171, 174, 177, 183,
179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193,
189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203,
199, 200, 201 204, 205, 206
}; };
#endif #endif
...@@ -783,7 +787,7 @@ while (YYID (0)) ...@@ -783,7 +787,7 @@ while (YYID (0))
#ifdef YYLEX_PARAM #ifdef YYLEX_PARAM
# define YYLEX yylex (&yylval, &yylloc, YYLEX_PARAM) # define YYLEX yylex (&yylval, &yylloc, YYLEX_PARAM)
#else #else
# define YYLEX yylex (&yylval, &yylloc) # define YYLEX yylex (&yylval, &yylloc, context)
#endif #endif
/* Enable debugging if requested. */ /* Enable debugging if requested. */
...@@ -2049,6 +2053,11 @@ yyreturn: ...@@ -2049,6 +2053,11 @@ yyreturn:
int yylex(YYSTYPE* lvalp, YYLTYPE* llocp, pp::Context* context)
{
return context->lex(lvalp, llocp);
}
void yyerror(YYLTYPE* llocp, pp::Context* context, const char* reason) void yyerror(YYLTYPE* llocp, pp::Context* context, const char* reason)
{ {
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment