Beginnings of a new preprocessor.

src/compiler/preprocessor/new/Context.cpp

+//
+// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#include "Context.h"
+
+namespace pp
+{
+
+Context::Context(int count, const char* const string[], const int length[],
+                 TokenVector* output)
+    : input(count, string, length),
+      output(output),
+      lexer(NULL)
+{
+}
+
+}  // namespace pp
+

src/compiler/preprocessor/new/Context.h

+//
+// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#ifndef COMPILER_PREPROCESSOR_CONTEXT_H_
+#define COMPILER_PREPROCESSOR_CONTEXT_H_
+
+#include "Input.h"
+#include "Macro.h"
+#include "Token.h"
+
+namespace pp
+{
+
+struct Context
+{
+    Context(int count, const char* const string[], const int length[],
+            TokenVector* output);
+
+    Input input;
+    TokenVector* output;
+
+    void* lexer;  // Lexer handle.
+    MacroSet macros;  // Defined macros.
+};
+
+}  // namespace pp
+#endif  // COMPILER_PREPROCESSOR_CONTEXT_H_
+

src/compiler/preprocessor/new/Input.cpp

+//
+// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#include "Input.h"
+
+#include <cstdio>
+
+#include "compiler/debug.h"
+
+namespace pp
+{
+
+Input::Input(int count, const char* const string[], const int length[])
+    : mCount(count),
+      mString(string),
+      mLength(length),
+      mIndex(-1),
+      mSize(0),
+      mError(kErrorNone),
+      mState(kStateInitial)
+{
+    ASSERT(mCount >= 0);
+    switchToNextString();
+}
+
+bool Input::eof() const
+{
+    ASSERT(mIndex <= mCount);
+    return mIndex == mCount;
+}
+
+int Input::read(char* buf, int bufSize)
+{
+    int nread = 0;
+    int startIndex = mIndex;
+    // Keep reading until the buffer is full or the current string is exhausted.
+    while ((mIndex == startIndex) && (nread < bufSize))
+    {
+        int c = getChar();
+        if (c == EOF)
+        {
+            if (mState == kStateBlockComment)
+                mError = kErrorUnexpectedEOF;
+            break;
+        }
+
+        switch (mState)
+        {
+          case kStateInitial:
+            if (c == '/')
+            {
+                // Potentially a comment.
+                switch (peekChar())
+                {
+                  case '/':
+                    getChar();  // Eat '/'.
+                    mState = kStateLineComment;
+                    break;
+                  case '*':
+                    getChar();  // Eat '*'.
+                    mState = kStateBlockComment;
+                    break;
+                  default:
+                    // Not a comment.
+                    buf[nread++] = c;
+                    break;
+                }
+            } else
+            {
+                buf[nread++] = c;
+            }
+            break;
+
+          case kStateLineComment:
+            if (c == '\n')
+            {
+                buf[nread++] = c;
+                mState = kStateInitial;
+            }
+            break;
+
+          case kStateBlockComment:
+            if (c == '*' && (peekChar() == '/'))
+            {
+                getChar();   // Eat '/'.
+                buf[nread++] = ' ';  // Replace comment with whitespace.
+                mState = kStateInitial;
+            } else if (c == '\n')
+            {
+                // Line breaks are never skipped.
+                buf[nread++] = c;
+            }
+            break;
+
+          default:
+            ASSERT(false);
+            break;
+        }
+    }
+
+    return nread;
+}
+
+int Input::getChar()
+{
+    if (eof()) return EOF;
+
+    const char* str = mString[mIndex];
+    int c = str[mSize++];
+
+    // Switch to next string if the current one is fully read.
+    int length = stringLength(mIndex);
+    // We never read from empty string.
+    ASSERT(length != 0);
+    if (((length < 0) && (str[mSize] == '\0')) ||
+        ((length > 0) && (mSize == length)))
+        switchToNextString();
+
+    return c;
+}
+
+int Input::peekChar()
+{
+    // Save the current read position.
+    int index = mIndex;
+    int size = mSize;
+    int c = getChar();
+
+    // Restore read position.
+    mIndex = index;
+    mSize = size;
+    return c;
+}
+
+void Input::switchToNextString()
+{
+    ASSERT(mIndex < mCount);
+
+    mSize = 0;
+    do
+    {
+        ++mIndex;
+    } while (!eof() && isStringEmpty(mIndex));
+}
+
+bool Input::isStringEmpty(int index)
+{
+    ASSERT(index < mCount);
+
+    const char* str = mString[mIndex];
+    int length = stringLength(mIndex);
+    return (length == 0) || ((length < 0) && (str[0] == '\0'));
+}
+
+int Input::stringLength(int index)
+{
+    ASSERT(index < mCount);
+    return mLength ? mLength[index] : -1;
+}
+
+}  // namespace pp
+

src/compiler/preprocessor/new/Input.h

+//
+// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#ifndef COMPILER_PREPROCESSOR_INPUT_H_
+#define COMPILER_PREPROCESSOR_INPUT_H_
+
+namespace pp
+{
+
+// Reads the given set of strings into input buffer.
+// Strips comments.
+class Input
+{
+  public:
+    Input(int count, const char* const string[], const int length[]);
+
+    enum Error
+    {
+        kErrorNone,
+        kErrorUnexpectedEOF
+    };
+    Error error() const { return mError; }
+
+    // Returns the index of string currently being scanned.
+    int stringIndex() const { return mIndex; }
+    // Returns true if EOF has reached.
+    bool eof() const;
+
+    // Reads up to bufSize characters into buf.
+    // Returns the number of characters read.
+    // It replaces each comment by a whitespace. It reads only one string
+    // at a time so that the lexer has opportunity to update the string number
+    // for meaningful diagnostic messages.
+    int read(char* buf, int bufSize);
+
+private:
+    enum State
+    {
+        kStateInitial,
+        kStateLineComment,
+        kStateBlockComment
+    };
+
+    int getChar();
+    int peekChar();
+    // Switches input buffer to the next non-empty string.
+    // This is called when the current string is fully read.
+    void switchToNextString();
+    // Returns true if the given string is empty.
+    bool isStringEmpty(int index);
+    // Return the length of the given string.
+    // Returns a negative value for null-terminated strings.
+    int stringLength(int index);
+
+    // Input.
+    int mCount;
+    const char* const* mString;
+    const int* mLength;
+
+    // Current read position.
+    int mIndex;   // Index of string currently being scanned.
+    int mSize;    // Size of string already scanned.
+
+    // Current error and state.
+    Error mError;
+    State mState;
+};
+
+}  // namespace pp
+#endif  // COMPILER_PREPROCESSOR_INPUT_H_
+

src/compiler/preprocessor/new/Macro.h

+//
+// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#ifndef COMPILER_PREPROCESSOR_MACRO_H_
+#define COMPILER_PREPROCESSOR_MACRO_H_
+
+#include <map>
+#include <string>
+
+#include "Token.h"
+
+namespace pp
+{
+
+struct Macro
+{
+    enum Type
+    {
+        kTypeObj,
+        kTypeFunc
+    };
+    Type type;
+    std::string identifier;
+    TokenVector parameters;
+    TokenVector replacements;
+};
+typedef std::map<std::string, Macro> MacroSet;
+
+}  // namespace pp
+#endif COMPILER_PREPROCESSOR_MACRO_H_
+

src/compiler/preprocessor/new/Preprocessor.cpp

+//
+// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#include "Preprocessor.h"
+
+#include "compiler/debug.h"
+#include "Context.h"
+
+namespace pp
+{
+
+bool Preprocessor::process(int count,
+                           const char* const string[],
+                           const int length[])
+{
+    ASSERT((count >=0) && (string != NULL));
+    if ((count < 0) || (string == NULL))
+        return false;
+
+    clearResults();
+    Context context(count, string, length, &mTokens);
+    if (!initLexer(&context))
+        return false;
+
+    bool success = parse(&context);
+
+    destroyLexer(&context);
+    return success;
+}
+
+void Preprocessor::clearResults()
+{
+    for (TokenVector::iterator i = mTokens.begin(); i != mTokens.end(); ++i)
+        delete (*i);
+
+    mTokens.clear();
+}
+
+}  // namespace pp
+

src/compiler/preprocessor/new/Preprocessor.h

+//
+// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#ifndef COMPILER_PREPROCESSOR_PREPROCESSOR_H_
+#define COMPILER_PREPROCESSOR_PREPROCESSOR_H_
+
+#include "common/angleutils.h"
+#include "Token.h"
+
+namespace pp
+{
+
+struct Context;
+
+class Preprocessor
+{
+  public:
+    Preprocessor() { }
+
+    bool process(int count, const char* const string[], const int length[]);
+
+    TokenIterator begin() const { return mTokens.begin(); }
+    TokenIterator end() const { return mTokens.end(); }
+
+  private:
+    DISALLOW_COPY_AND_ASSIGN(Preprocessor);
+
+    static bool initLexer(Context* context);
+    static void destroyLexer(Context* context);
+    static bool parse(Context* context);
+
+    void clearResults();
+
+    TokenVector mTokens;  // Output.
+};
+
+}  // namespace pp
+#endif  // COMPILER_PREPROCESSOR_PREPROCESSOR_H_
+

src/compiler/preprocessor/new/Token.cpp

+//
+// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#include "Token.h"
+
+static const int kLocationLineSize = 16;  // in bits.
+static const int kLocationLineMask = (1 << kLocationLineSize) - 1;
+
+namespace pp
+{
+
+Token::Token() : mLocation(-1), mType(-1), mValue(0)
+{
+}
+
+Token::Token(Location l, int t) : mLocation(l), mType(t)
+{
+}
+
+Token::Token(Location l, int t, const std::string& s) : mLocation(l), mType(t), mValue(s)
+{
+}
+
+Token::Location Token::encodeLocation(int line, int file)
+{
+    return (file << kLocationLineSize) | (line & kLocationLineMask);
+}
+
+void Token::decodeLocation(Location loc, int* line, int* file)
+{
+    if (file) *file = loc >> kLocationLineSize;
+    if (line) *line = loc & kLocationLineMask;
+}
+
+}  // namespace pp
+

src/compiler/preprocessor/new/Token.h

+//
+// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#ifndef COMPILER_PREPROCESSOR_TOKEN_H_
+#define COMPILER_PREPROCESSOR_TOKEN_H_
+
+#include <string>
+#include <vector>
+
+namespace pp
+{
+
+class Token
+{
+  public:
+    typedef int Location;
+
+    Token();
+    Token(Location l, int t);
+    Token(Location l, int t, const std::string& s);
+
+    static Location encodeLocation(int line, int file);
+    static void decodeLocation(Location loc, int* line, int* file);
+
+    Location location() const { return mLocation; }
+    int type() const { return mType; }
+    const std::string& value() const { return mValue; }
+
+  private:
+    Location mLocation;
+    int mType;
+    std::string mValue;
+};
+
+typedef std::vector<Token*> TokenVector;
+typedef TokenVector::const_iterator TokenIterator;
+
+}  // namepsace pp
+#endif  // COMPILER_PREPROCESSOR_TOKEN_H_
+

src/compiler/preprocessor/new/generate_parser.sh

+#!/bin/bash
+# Copyright (c) 2010 The ANGLE Project Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Generates GLSL ES preprocessor - pp_lex.cpp, pp_tab.h, and pp_tab.cpp
+
+run_flex()
+{
+input_file=$script_dir/$1.l
+output_source=$script_dir/$1_lex.cpp
+flex --noline --nounistd --outfile=$output_source $input_file
+}
+
+run_bison()
+{
+input_file=$script_dir/$1.y
+output_header=$script_dir/$1_tab.h
+output_source=$script_dir/$1_tab.cpp
+bison --no-lines --skeleton=yacc.c --defines=$output_header --output=$output_source $input_file
+}
+
+script_dir=$(dirname $0)
+
+# Generate preprocessor
+run_flex pp
+run_bison pp

src/compiler/preprocessor/new/pp.l

+/*
+//
+// Copyright (c) 2002-2011 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+This file contains the Lex specification for GLSL ES preprocessor.
+Based on Microsoft Visual Studio 2010 Preprocessor Grammar:
+http://msdn.microsoft.com/en-us/library/2scxys89.aspx
+
+IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
+*/
+
+%top{
+//
+// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+// This file is auto-generated by generate_parser.sh. DO NOT EDIT!
+}
+
+%{
+#include "compiler/debug.h"
+#include "Context.h"
+#include "pp_tab.h"
+#include "Preprocessor.h"
+
+#define YY_USER_ACTION                        \
+    do {                                      \
+        yylloc->first_line = yylineno;        \
+        yylloc->first_column = yycolumn + 1;  \
+        yycolumn += yyleng;                   \
+    } while(0);
+    
+#define YY_INPUT(buf, result, maxSize) \
+    result = readInput(yyextra, buf, maxSize);
+
+static int readInput(pp::Context* context, char* buf, int maxSize);
+%}
+
+%option noyywrap nounput never-interactive
+%option yylineno reentrant bison-bridge bison-locations
+%option stack
+%option prefix="pp"
+%option extra-type="pp::Context*"
+
+HSPACE      [ \t]
+HASH        ^{HSPACE}*#{HSPACE}*
+IDENTIFIER  [_a-zA-Z][_a-zA-Z0-9]*
+PUNCTUATOR  [][<>(){}.+-/*%^|&~=!:;,?]
+
+DECIMAL_CONSTANT      [1-9][0-9]*
+OCTAL_CONSTANT        0[0-7]*
+HEXADECIMAL_CONSTANT  0[xX][0-9a-fA-F]+
+
+%%
+
+{HASH} { return HASH; }
+
+{HASH}define { return HASH_DEFINE_OBJ; }
+{HASH}define{HSPACE}+/{IDENTIFIER}"(" { return HASH_DEFINE_FUNC; }
+{HASH}undef { return HASH_UNDEF; }
+
+{HASH}if { return HASH_IF; }
+{HASH}ifdef { return HASH_IFDEF; }
+{HASH}ifndef { return HASH_IFNDEF; }
+{HASH}else { return HASH_ELSE; }
+{HASH}elif { return HASH_ELIF; }
+{HASH}endif { return HASH_ENDIF; }
+"defined" { return DEFINED; }
+
+{HASH}error { return HASH_ERROR; }
+{HASH}pragma { return HASH_PRAGMA; }
+{HASH}extension { return HASH_EXTENSION; }
+{HASH}version { return HASH_VERSION; }
+{HASH}line { return HASH_LINE; }
+
+{IDENTIFIER} {
+    yylval->tval = new pp::Token(yylineno, IDENTIFIER, std::string(yytext, yyleng));
+    return IDENTIFIER;
+}
+
+{DECIMAL_CONSTANT}|{OCTAL_CONSTANT}|{HEXADECIMAL_CONSTANT} {
+    yylval->tval = new pp::Token(yylineno, INTEGER_CONSTANT, std::string(yytext, yyleng));
+    return INTEGER_CONSTANT;
+}
+
+{PUNCTUATOR} { return yytext[0]; }
+
+{HSPACE}+ { return SPACE; }
+
+\n {
+    ++yylineno; yycolumn = 0;
+    return yytext[0];
+}
+
+<*><<EOF>>     { yyterminate(); }
+
+%%
+
+int readInput(pp::Context* context, char* buf, int maxSize)
+{
+    yyscan_t lexer = context->lexer;
+    ASSERT(lexer);
+
+    int nread = YY_NULL;
+    while (!context->input.eof() &&
+           (context->input.error() == pp::Input::kErrorNone) &&
+           (nread == YY_NULL))
+    {
+        int line = 0, file = 0;
+        pp::Token::decodeLocation(yyget_lineno(lexer), &line, &file);
+        file = context->input.stringIndex();
+        yyset_lineno(pp::Token::encodeLocation(line, file), lexer);
+
+        nread = context->input.read(buf, maxSize);
+
+        if (context->input.error() == pp::Input::kErrorUnexpectedEOF)
+        {
+            // TODO(alokp): Report error.
+        }
+    }
+    return nread;
+}
+
+namespace pp {
+
+bool Preprocessor::initLexer(Context* context)
+{
+    ASSERT(context->lexer == NULL);
+
+    yyscan_t lexer = 0;
+    if (yylex_init_extra(context, &lexer))
+        return false;
+
+    context->lexer = lexer;
+    yyrestart(0, lexer);
+    return true;
+}
+
+void Preprocessor::destroyLexer(Context* context)
+{
+    ASSERT(context->lexer);
+
+    yylex_destroy(context->lexer);
+    context->lexer = 0;
+}
+
+}  // namespace pp
+

src/compiler/preprocessor/new/pp.y

+/*
+//
+// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+This file contains the Yacc grammar for GLSL ES preprocessor.
+Based on Microsoft Visual Studio 2010 Preprocessor Grammar:
+http://msdn.microsoft.com/en-us/library/2scxys89.aspx
+
+IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_glsl_parser.sh,
+WHICH GENERATES THE GLSL ES PARSER.
+*/
+
+%{
+//
+// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+// This file is auto-generated by generate_glslang_parser.sh. DO NOT EDIT!
+
+#include "Context.h"
+#include "Preprocessor.h"
+
+#define YYLEX_PARAM context->lexer
+#define YYDEBUG 1
+%}
+
+%pure-parser
+%name-prefix="pp"
+%locations
+%parse-param {pp::Context* context}
+
+%union {
+    int ival;
+    pp::Token* tval;
+    pp::TokenVector* tlist;
+}
+
+%{
+extern int yylex(YYSTYPE* lvalp, YYLTYPE* llocp, void* lexer);
+static void yyerror(YYLTYPE* llocp,
+                    pp::Context* context,
+                    const char* reason);
+
+static void defineMacro(pp::Context* context,
+                        YYLTYPE* llocp,
+                        pp::Macro::Type type,
+                        pp::Token* identifier,
+                        pp::TokenVector* parameters,
+                        pp::TokenVector* replacements);
+static void undefineMacro(pp::Context* context, pp::Token* identifier);
+static bool isMacroDefined(pp::Context* context, pp::Token* identifier);
+static void pushConditionalBlock(pp::Context* context, bool condition);
+static void popConditionalBlock(pp::Context* context);
+%}
+
+%token HASH HASH_DEFINE_OBJ HASH_DEFINE_FUNC HASH_UNDEF
+%token HASH_IF HASH_IFDEF HASH_IFNDEF HASH_ELSE HASH_ELIF HASH_ENDIF DEFINED
+%token HASH_ERROR HASH_PRAGMA HASH_EXTENSION HASH_VERSION HASH_LINE
+%token SPACE
+%token <tval> IDENTIFIER INTEGER_CONSTANT FLOAT_CONSTANT
+%type <ival> operator
+%type <tval> conditional_token token
+%type <tlist> text_line parameter_list replacement_token_list conditional_token_list token_list
+%%
+
+input
+    : /* empty */
+    | input line
+;
+
+line
+    : text_line
+    | control_line
+;
+
+text_line
+    : '\n' { $$ = NULL; }
+    | token_list '\n' { $$ = $1; }
+;
+
+control_line
+    : HASH '\n'
+    | HASH_DEFINE_OBJ IDENTIFIER replacement_token_list '\n' {
+        defineMacro(context, & @2, pp::Macro::kTypeObj, $2, NULL, $3);
+    }
+    | HASH_DEFINE_FUNC IDENTIFIER '(' parameter_list ')' replacement_token_list '\n' {
+        defineMacro(context, & @2, pp::Macro::kTypeFunc, $2, $4, $6);
+    }
+    | HASH_UNDEF IDENTIFIER '\n' {
+        undefineMacro(context, $2);
+    }
+    | HASH_IF conditional_token_list '\n' {
+        pushConditionalBlock(context, $2 ? true : false);
+    }
+    | HASH_IFDEF IDENTIFIER '\n' {
+        pushConditionalBlock(context, isMacroDefined(context, $2));
+    }
+    | HASH_IFNDEF IDENTIFIER '\n' {
+        pushConditionalBlock(context, !isMacroDefined(context, $2));
+    }
+    | HASH_ELIF conditional_token_list '\n' {
+    }
+    | HASH_ELSE '\n' {
+    }
+    | HASH_ENDIF '\n' {
+        popConditionalBlock(context);
+    }
+    | HASH_ERROR '\n'
+    | HASH_PRAGMA '\n'
+    | HASH_EXTENSION '\n'
+    | HASH_VERSION '\n'
+    | HASH_LINE '\n'
+;
+
+replacement_token_list
+    : /* empty */ { $$ = NULL }
+    | token_list
+;
+
+conditional_token_list
+    : conditional_token {
+        $$ = new pp::TokenVector;
+        $$->push_back($1);
+    }
+    | conditional_token_list conditional_token {
+        $$ = $1;
+        $$->push_back($2);
+    }
+;
+
+conditional_token
+    : DEFINED IDENTIFIER {
+    }
+    | DEFINED '(' IDENTIFIER ')' {
+    }
+    | token
+;
+
+parameter_list
+    : /* empty */ { $$ = NULL; }
+    | IDENTIFIER {
+        $$ = new pp::TokenVector;
+        $$->push_back($1);
+    }
+    | parameter_list ',' IDENTIFIER {
+        $$ = $1;
+        $$->push_back($3);
+    }
+;
+
+token_list
+    : token {
+        //context->ppData.skipWS = false;
+        $$ = new pp::TokenVector;
+        $$->push_back($1);
+    }
+    | token_list token {
+        $$ = $1;
+        $$->push_back($2);
+    }
+;
+
+token
+    : IDENTIFIER
+    | INTEGER_CONSTANT
+    | FLOAT_CONSTANT
+    | SPACE { $$ = new pp::Token(@1.first_line, SPACE); }
+    | operator { $$ = new pp::Token(@1.first_line, $1); }
+;
+
+operator
+    : '[' { $$ = '['; }
+    | ']' { $$ = ']'; }
+    | '<' { $$ = '<'; }
+    | '>' { $$ = '>'; }
+    | '(' { $$ = '('; }
+    | ')' { $$ = ')'; }
+    | '{' { $$ = '{'; }
+    | '}' { $$ = '}'; }
+    | '.' { $$ = '.'; }
+    | '+' { $$ = '+'; }
+    | '-' { $$ = '-'; }
+    | '/' { $$ = '/'; }
+    | '*' { $$ = '*'; }
+    | '%' { $$ = '%'; }
+    | '^' { $$ = '^'; }
+    | '|' { $$ = '|'; }
+    | '&' { $$ = '&'; }
+    | '~' { $$ = '~'; }
+    | '=' { $$ = '='; }
+    | '!' { $$ = '!'; }
+    | ':' { $$ = ':'; }
+    | ';' { $$ = ';'; }
+    | ',' { $$ = ','; }
+    | '?' { $$ = '?'; }
+;
+
+%%
+
+void yyerror(YYLTYPE* llocp, pp::Context* context, const char* reason)
+{
+}
+
+void defineMacro(pp::Context* context,
+                 YYLTYPE* llocp,
+                 pp::Macro::Type type,
+                 pp::Token* identifier,
+                 pp::TokenVector* parameters,
+                 pp::TokenVector* replacements)
+{
+}
+
+void undefineMacro(pp::Context* context, pp::Token* identifier)
+{
+}
+
+bool isMacroDefined(pp::Context* context, pp::Token* identifier)
+{
+    return false;
+}
+
+void pushConditionalBlock(pp::Context* context, bool condition)
+{
+}
+
+void popConditionalBlock(pp::Context* context)
+{
+}
+
+namespace pp {
+bool Preprocessor::parse(Context* context)
+{
+    yydebug = 1;
+    return yyparse(context) == 0 ? true : false;
+}
+}  // namespace pp
+

src/compiler/preprocessor/new/pp_tab.h

+/* A Bison parser, made by GNU Bison 2.3.  */
+
+/* Skeleton interface for Bison's Yacc-like parsers in C
+
+   Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006
+   Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor,
+   Boston, MA 02110-1301, USA.  */
+
+/* As a special exception, you may create a larger work that contains
+   part or all of the Bison parser skeleton and distribute that work
+   under terms of your choice, so long as that work isn't itself a
+   parser generator using the skeleton or a modified version thereof
+   as a parser skeleton.  Alternatively, if you modify or redistribute
+   the parser skeleton itself, you may (at your option) remove this
+   special exception, which will cause the skeleton and the resulting
+   Bison output files to be licensed under the GNU General Public
+   License without this special exception.
+
+   This special exception was added by the Free Software Foundation in
+   version 2.2 of Bison.  */
+
+/* Tokens.  */
+#ifndef YYTOKENTYPE
+# define YYTOKENTYPE
+   /* Put the tokens into the symbol table, so that GDB and other debuggers
+      know about them.  */
+   enum yytokentype {
+     HASH = 258,
+     HASH_DEFINE_OBJ = 259,
+     HASH_DEFINE_FUNC = 260,
+     HASH_UNDEF = 261,
+     HASH_IF = 262,
+     HASH_IFDEF = 263,
+     HASH_IFNDEF = 264,
+     HASH_ELSE = 265,
+     HASH_ELIF = 266,
+     HASH_ENDIF = 267,
+     DEFINED = 268,
+     HASH_ERROR = 269,
+     HASH_PRAGMA = 270,
+     HASH_EXTENSION = 271,
+     HASH_VERSION = 272,
+     HASH_LINE = 273,
+     SPACE = 274,
+     IDENTIFIER = 275,
+     INTEGER_CONSTANT = 276,
+     FLOAT_CONSTANT = 277
+   };
+#endif
+/* Tokens.  */
+#define HASH 258
+#define HASH_DEFINE_OBJ 259
+#define HASH_DEFINE_FUNC 260
+#define HASH_UNDEF 261
+#define HASH_IF 262
+#define HASH_IFDEF 263
+#define HASH_IFNDEF 264
+#define HASH_ELSE 265
+#define HASH_ELIF 266
+#define HASH_ENDIF 267
+#define DEFINED 268
+#define HASH_ERROR 269
+#define HASH_PRAGMA 270
+#define HASH_EXTENSION 271
+#define HASH_VERSION 272
+#define HASH_LINE 273
+#define SPACE 274
+#define IDENTIFIER 275
+#define INTEGER_CONSTANT 276
+#define FLOAT_CONSTANT 277
+
+
+
+
+#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
+typedef union YYSTYPE
+
+{
+    int ival;
+    pp::Token* tval;
+    pp::TokenVector* tlist;
+}
+/* Line 1489 of yacc.c.  */
+
+	YYSTYPE;
+# define yystype YYSTYPE /* obsolescent; will be withdrawn */
+# define YYSTYPE_IS_DECLARED 1
+# define YYSTYPE_IS_TRIVIAL 1
+#endif
+
+
+
+#if ! defined YYLTYPE && ! defined YYLTYPE_IS_DECLARED
+typedef struct YYLTYPE
+{
+  int first_line;
+  int first_column;
+  int last_line;
+  int last_column;
+} YYLTYPE;
+# define yyltype YYLTYPE /* obsolescent; will be withdrawn */
+# define YYLTYPE_IS_DECLARED 1
+# define YYLTYPE_IS_TRIVIAL 1
+#endif
+
+