Using yy_scan_string, which flushes the old buffer does not work. GLSL requires…

src/compiler/preprocessor/new/Input.cpp

@@ -6,11 +6,55 @@
 
 #include "Input.h"
 
+#include <algorithm>
+#include <cassert>
+
 namespace pp
 {
 
-Input::Input() : count(0), string(0), length(0), index(0), buffer(0)
+Input::Input() : mCount(0), mString(0), mLength(0)
+{
+}
+
+Input::Input(int count, const char* const string[], const int length[]) :
+    mCount(count),
+    mString(string),
+    mLength(0)
 {
+    assert(mCount >= 0);
+    mLength = mCount > 0 ? new int[mCount] : 0;
+    for (int i = 0; i < mCount; ++i)
+    {
+        mLength[i] = length ? length[i] : -1;
+        if (mLength[i] < 0)
+            mLength[i] = strlen(mString[i]);
+    }
+}
+
+Input::~Input()
+{
+    if (mLength) delete [] mLength;
+}
+
+int Input::read(char* buf, int maxSize)
+{
+    int nRead = 0;
+    while ((nRead < maxSize) && (mReadLoc.sIndex < mCount))
+    {
+        int size = mLength[mReadLoc.sIndex] - mReadLoc.cIndex;
+        size = std::min(size, maxSize);
+        memcpy(buf + nRead, mString[mReadLoc.sIndex] + mReadLoc.cIndex, size);
+        nRead += size;
+        mReadLoc.cIndex += size;
+
+        // Advance string if we reached the end of current string.
+        if (mReadLoc.cIndex == mLength[mReadLoc.sIndex])
+        {
+            ++mReadLoc.sIndex;
+            mReadLoc.cIndex = 0;
+        }
+    }
+    return nRead;
 }
 
 }  // namespace pp

src/compiler/preprocessor/new/Input.h

@@ -7,22 +7,43 @@
 #ifndef COMPILER_PREPROCESSOR_INPUT_H_
 #define COMPILER_PREPROCESSOR_INPUT_H_
 
+#include "pp_utils.h"
+
 namespace pp
 {
 
-// Holds lexer input.
-struct Input
+// Holds and reads input for Lexer.
+class Input
 {
-    // Input.
-    int count;
-    const char* const* string;
-    const int* length;
+  public:
+    Input();
+    Input(int count, const char* const string[], const int length[]);
+    ~Input();
 
-    // Current read position.
-    int index;  // Index of string currently being scanned.
-    void* buffer;  // Current buffer handle.
+    int count() const { return mCount; }
+    const char* string(int index) const { return mString[index]; }
+    int length(int index) const { return mLength[index]; }
 
-    Input();
+    int read(char* buf, int maxSize);
+
+    struct Location
+    {
+        int sIndex;  // String index;
+        int cIndex;  // Char index.
+
+        Location() : sIndex(0), cIndex(0) { }
+    };
+    const Location& readLoc() const { return mReadLoc; }
+
+  private:
+    PP_DISALLOW_COPY_AND_ASSIGN(Input);
+
+    // Input.
+    int mCount;
+    const char* const* mString;
+    int* mLength;
+
+    Location mReadLoc;
 };
 
 }  // namespace pp

src/compiler/preprocessor/new/Lexer.cpp

@@ -18,20 +18,14 @@ Lexer::Lexer() : mHandle(0)
 Lexer::~Lexer()
 {
     destroyLexer();
-
-    // Make sure the lexer and associated buffer are deleted.
-    assert(mHandle == 0);
-    assert(mInput.buffer == 0);
 }
 
 bool Lexer::init(int count, const char* const string[], const int length[])
 {
-    assert((count >= 0) && string);
-
-    mInput.count = count;
-    mInput.string = string;
-    mInput.length = length;
+    if (count < 0) return false;
+    if ((count > 0) && (string == 0)) return false;
 
+    mContext.input.reset(new Input(count, string, length));
     return initLexer();
 }
 

src/compiler/preprocessor/new/Lexer.h

@@ -7,6 +7,8 @@
 #ifndef COMPILER_PREPROCESSOR_LEXER_H_
 #define COMPILER_PREPROCESSOR_LEXER_H_
 
+#include <memory>
+
 #include "Input.h"
 #include "pp_utils.h"
 
@@ -18,11 +20,19 @@ struct Token;
 class Lexer
 {
   public:
+    struct Context
+    {
+        std::auto_ptr<Input> input;
+        // The location where yytext points to. Token location should track
+        // scanLoc instead of Input::mReadLoc because they may not be the same
+        // if text is buffered up in the lexer input buffer.
+        Input::Location scanLoc;
+    };
+
     Lexer();
     ~Lexer();
 
     bool init(int count, const char* const string[], const int length[]);
-
     int lex(Token* token);
 
   private:
@@ -31,7 +41,7 @@ class Lexer
     void destroyLexer();
 
     void* mHandle;  // Lexer handle.
-    Input mInput;  // Input buffer.
+    Context mContext;  // Lexer extra.
 };
 
 }  // namespace pp

src/compiler/preprocessor/new/Preprocessor.h

@@ -18,7 +18,17 @@ class Preprocessor
   public:
     Preprocessor() { }
 
+    // count: specifies the number of elements in the string and length arrays.
+    // string: specifies an array of pointers to strings.
+    // length: specifies an array of string lengths.
+    // If length is NULL, each string is assumed to be null terminated.
+    // If length is a value other than NULL, it points to an array containing
+    // a string length for each of the corresponding elements of string.
+    // Each element in the length array may contain the length of the
+    // corresponding string or a value less than 0 to indicate that the string
+    // is null terminated.
     bool init(int count, const char* const string[], const int length[]);
+
     int lex(Token* token);
 
   private:

src/compiler/preprocessor/new/pp.l

@@ -23,50 +23,44 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
 }
 
 %{
-#include "Input.h"
 #include "Lexer.h"
 #include "Token.h"
 
-// Token location (file and line) is encoded into a single int so that
-// we can use yylineno macro.
-
-const unsigned int kLocationLineSize = 16;  // in bits.
-const unsigned int kLocationLineMask = (1 << kLocationLineSize) - 1;
-
-static int location(int file, int line)
-{
-    return (file << kLocationLineSize) | (line & kLocationLineMask);
-}
-
-static int locationFile(int loc)
-{
-    return loc >> kLocationLineSize;
-}
-
-static int locationLine(int loc)
-{
-    return loc & kLocationLineMask;
-}
-
 typedef std::string YYSTYPE;
 typedef pp::Token::Location YYLTYPE;
 
+// Use the unused yycolumn variable to track file (string) number.
+#define yyfileno yycolumn
+
+#define YY_USER_INIT  \
+    do {              \
+        yyfileno = 0; \
+        yylineno = 1; \
+    } while(0);
+
 #define YY_USER_ACTION                                            \
     do {                                                          \
-        yylloc->file = locationFile(yylineno);  \
-        yylloc->line = locationLine(yylineno);  \
+        pp::Input* input = yyextra->input.get();                  \
+        pp::Input::Location* scanLoc = &yyextra->scanLoc;         \
+        while (scanLoc->cIndex >= input->length(scanLoc->sIndex)) \
+        {                                                         \
+            scanLoc->cIndex -= input->length(scanLoc->sIndex++);  \
+            ++yyfileno; yylineno = 1;                             \
+        }                                                         \
+        yylloc->file = yyfileno;                                  \
+        yylloc->line = yylineno;                                  \
+        scanLoc->cIndex += yyleng;                                \
     } while(0);
 
-// Suppress the default implementation of YY_INPUT which generated
-// compiler warnings.
-#define YY_INPUT
+#define YY_INPUT(buf, result, maxSize) \
+    result = yyextra->input->read(buf, maxSize);
 
 %}
 
-%option nounput never-interactive
+%option noyywrap nounput never-interactive
 %option reentrant bison-bridge bison-locations
 %option prefix="pp"
-%option extra-type="pp::Input*"
+%option extra-type="pp::Lexer::Context*"
 %x COMMENT
 
 NEWLINE     \n|\r|\r\n
@@ -159,42 +153,6 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 
 %%
 
-int ppwrap(yyscan_t scanner)
-{
-    pp::Input* input = yyget_extra(scanner);
-
-    int file = -1;
-    // Delete the current buffer before switching to the next one.
-    YY_BUFFER_STATE buffer = static_cast<YY_BUFFER_STATE>(input->buffer);
-    if (buffer != NULL)
-    {
-        // Save the current token location before they get lost when the
-        // current input buffer is deleted.
-        file = locationFile(yyget_lineno(scanner));
-
-        yy_delete_buffer(buffer, scanner);
-        input->buffer = NULL;
-    }
-
-    int index = std::min(input->index + 1, input->count);
-    if (index == input->count)
-        return 1;  // EOF reached.
-
-    int length = input->length ? input->length[index] : -1;
-    if (length < 0)  // NULL terminated string.
-        buffer = yy_scan_string(input->string[index], scanner);
-    else
-        buffer = yy_scan_bytes(input->string[index], length, scanner);
-
-    // Increment file number and reset line number.
-    // This can be done only after we have created a new input buffer.
-    yyset_lineno(location(file + 1, 1), scanner);
-
-    input->index = index;
-    input->buffer = buffer;
-    return 0;
-}
-
 namespace pp {
 
 int Lexer::lex(Token* token)
@@ -213,13 +171,10 @@ int Lexer::lex(Token* token)
 
 bool Lexer::initLexer()
 {
-    if ((mHandle == NULL) && yylex_init_extra(&mInput, &mHandle))
+    if ((mHandle == NULL) && yylex_init_extra(&mContext, &mHandle))
         return false;
 
-    // Setup first scan string.
-    mInput.index = -1;
-    ppwrap(mHandle);
-
+    yyrestart(0, mHandle);
     return true;
 }
 
@@ -228,13 +183,6 @@ void Lexer::destroyLexer()
     if (mHandle == NULL)
         return;
 
-    YY_BUFFER_STATE buffer = static_cast<YY_BUFFER_STATE>(mInput.buffer);
-    if (buffer != NULL)
-    {
-        yy_delete_buffer(buffer, mHandle);
-        mInput.buffer = NULL;
-    }
- 
     yylex_destroy(mHandle);
     mHandle = NULL;
 }

src/compiler/preprocessor/new/pp_lex.cpp

@@ -334,6 +334,9 @@ void ppfree (void * ,yyscan_t yyscanner );
 
 /* Begin user sect3 */
 
+#define ppwrap(n) 1
+#define YY_SKIP_YYWRAP
+
 typedef unsigned char YY_CHAR;
 
 typedef int yy_state_type;
@@ -511,48 +514,42 @@ http://msdn.microsoft.com/en-us/library/2scxys89.aspx
 IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
 */
 
-#include "Input.h"
 #include "Lexer.h"
 #include "Token.h"
 
-// Token location (file and line) is encoded into a single int so that
-// we can use yylineno macro.
-
-const unsigned int kLocationLineSize = 16;  // in bits.
-const unsigned int kLocationLineMask = (1 << kLocationLineSize) - 1;
-
-static int location(int file, int line)
-{
-    return (file << kLocationLineSize) | (line & kLocationLineMask);
-}
-
-static int locationFile(int loc)
-{
-    return loc >> kLocationLineSize;
-}
-
-static int locationLine(int loc)
-{
-    return loc & kLocationLineMask;
-}
-
 typedef std::string YYSTYPE;
 typedef pp::Token::Location YYLTYPE;
 
+// Use the unused yycolumn variable to track file (string) number.
+#define yyfileno yycolumn
+
+#define YY_USER_INIT  \
+    do {              \
+        yyfileno = 0; \
+        yylineno = 1; \
+    } while(0);
+
 #define YY_USER_ACTION                                            \
     do {                                                          \
-        yylloc->file = locationFile(yylineno);  \
-        yylloc->line = locationLine(yylineno);  \
+        pp::Input* input = yyextra->input.get();                  \
+        pp::Input::Location* scanLoc = &yyextra->scanLoc;         \
+        while (scanLoc->cIndex >= input->length(scanLoc->sIndex)) \
+        {                                                         \
+            scanLoc->cIndex -= input->length(scanLoc->sIndex++);  \
+            ++yyfileno; yylineno = 1;                             \
+        }                                                         \
+        yylloc->file = yyfileno;                                  \
+        yylloc->line = yylineno;                                  \
+        scanLoc->cIndex += yyleng;                                \
     } while(0);
 
-// Suppress the default implementation of YY_INPUT which generated
-// compiler warnings.
-#define YY_INPUT
+#define YY_INPUT(buf, result, maxSize) \
+    result = yyextra->input->read(buf, maxSize);
 
 #define INITIAL 0
 #define COMMENT 1
 
-#define YY_EXTRA_TYPE pp::Input*
+#define YY_EXTRA_TYPE pp::Lexer::Context*
 
 /* Holds the entire state of the reentrant scanner. */
 struct yyguts_t
@@ -2174,42 +2171,6 @@ void ppfree (void * ptr , yyscan_t yyscanner)
 
 #define YYTABLES_NAME "yytables"
 
-int ppwrap(yyscan_t scanner)
-{
-    pp::Input* input = ppget_extra(scanner);
-
-    int file = -1;
-    // Delete the current buffer before switching to the next one.
-    YY_BUFFER_STATE buffer = static_cast<YY_BUFFER_STATE>(input->buffer);
-    if (buffer != NULL)
-    {
-        // Save the current token location before they get lost when the
-        // current input buffer is deleted.
-        file = locationFile(ppget_lineno(scanner));
-
-        pp_delete_buffer(buffer,scanner);
-        input->buffer = NULL;
-    }
-
-    int index = std::min(input->index + 1, input->count);
-    if (index == input->count)
-        return 1;  // EOF reached.
-
-    int length = input->length ? input->length[index] : -1;
-    if (length < 0)  // NULL terminated string.
-        buffer = pp_scan_string(input->string[index],scanner);
-    else
-        buffer = pp_scan_bytes(input->string[index],length,scanner);
-
-    // Increment file number and reset line number.
-    // This can be done only after we have created a new input buffer.
-    ppset_lineno(location(file + 1, 1),scanner);
-
-    input->index = index;
-    input->buffer = buffer;
-    return 0;
-}
-
 namespace pp {
 
 int Lexer::lex(Token* token)
@@ -2228,13 +2189,10 @@ int Lexer::lex(Token* token)
 
 bool Lexer::initLexer()
 {
-    if ((mHandle == NULL) && pplex_init_extra(&mInput,&mHandle))
+    if ((mHandle == NULL) && pplex_init_extra(&mContext,&mHandle))
         return false;
 
-    // Setup first scan string.
-    mInput.index = -1;
-    ppwrap(mHandle);
-
+    pprestart(0,mHandle);
     return true;
 }
 
@@ -2243,13 +2201,6 @@ void Lexer::destroyLexer()
     if (mHandle == NULL)
         return;
 
-    YY_BUFFER_STATE buffer = static_cast<YY_BUFFER_STATE>(mInput.buffer);
-    if (buffer != NULL)
-    {
-        pp_delete_buffer(buffer,mHandle);
-        mInput.buffer = NULL;
-    }
- 
     pplex_destroy(mHandle);
     mHandle = NULL;
 }

tests/build_tests.gyp

@@ -31,6 +31,7 @@
         'preprocessor_tests/char_test.cpp',
         'preprocessor_tests/comment_test.cpp',
         'preprocessor_tests/identifier_test.cpp',
+        'preprocessor_tests/input_test.cpp',
         'preprocessor_tests/location_test.cpp',
         'preprocessor_tests/number_test.cpp',
         'preprocessor_tests/operator_test.cpp',

tests/preprocessor_tests/input_test.cpp

+//
+// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#include "gtest/gtest.h"
+#include "Preprocessor.h"
+#include "Token.h"
+
+TEST(InputTest, NegativeCount)
+{
+    pp::Preprocessor preprocessor;
+    EXPECT_FALSE(preprocessor.init(-1, NULL, NULL));
+}
+
+TEST(InputTest, ZeroCount)
+{
+    pp::Token token;
+    pp::Preprocessor preprocessor;
+    EXPECT_TRUE(preprocessor.init(0, NULL, NULL));
+    EXPECT_EQ(pp::Token::LAST, preprocessor.lex(&token));
+    EXPECT_EQ(pp::Token::LAST, token.type);
+}
+
+TEST(InputTest, NullString)
+{
+    pp::Preprocessor preprocessor;
+    EXPECT_FALSE(preprocessor.init(1, NULL, NULL));
+}
+
+TEST(InputTest, DefaultConstructor)
+{
+    pp::Input input;
+    EXPECT_EQ(0, input.count());
+    EXPECT_EQ(0, input.read(NULL, 1));
+}
+
+TEST(InputTest, NullLength)
+{
+    char* str[] = {"foo"};
+    pp::Input input(1, str, NULL);
+    EXPECT_EQ(3, input.length(0));
+}
+
+TEST(InputTest, NegativeLength)
+{
+    char* str[] = {"foo"};
+    int length[] = {-1};
+    pp::Input input(1, str, length);
+    EXPECT_EQ(3, input.length(0));
+}
+
+TEST(InputTest, ActualLength)
+{
+    char* str[] = {"foobar"};
+    int length[] = {3};
+    pp::Input input(1, str, length);
+    // Note that strlen(str[0]) != length[0].
+    // Even then Input should just accept any non-negative number.
+    EXPECT_EQ(length[0], input.length(0));
+}
+
+TEST(InputTest, String)
+{
+    char* str[] = {"foo"};
+    pp::Input input(1, str, NULL);
+    EXPECT_STREQ(str[0], input.string(0));
+}
+
+TEST(InputTest, ReadSingleString)
+{
+    int count = 1;
+    char* str[] = {"foo"};
+    char buf[4] = {'\0', '\0', '\0', '\0'};
+
+    int maxSize = 1;
+    pp::Input input1(count, str, NULL);
+    EXPECT_EQ(1, input1.read(buf, maxSize));
+    EXPECT_EQ('f', buf[0]);
+    EXPECT_EQ(1, input1.read(buf, maxSize));
+    EXPECT_EQ('o', buf[0]);
+    EXPECT_EQ(1, input1.read(buf, maxSize));
+    EXPECT_EQ('o', buf[0]);
+    EXPECT_EQ(0, input1.read(buf, maxSize));
+
+    maxSize = 2;
+    pp::Input input2(count, str, NULL);
+    EXPECT_EQ(2, input2.read(buf, maxSize));
+    EXPECT_STREQ("fo", buf);
+    EXPECT_EQ(1, input2.read(buf, maxSize));
+    EXPECT_EQ('o', buf[0]);
+    EXPECT_EQ(0, input2.read(buf, maxSize));
+
+    maxSize = 3;
+    pp::Input input3(count, str, NULL);
+    EXPECT_EQ(3, input3.read(buf, maxSize));
+    EXPECT_STREQ("foo", buf);
+    EXPECT_EQ(0, input3.read(buf, maxSize));
+
+    maxSize = 4;
+    pp::Input input4(count, str, NULL);
+    EXPECT_EQ(3, input4.read(buf, maxSize));
+    EXPECT_STREQ("foo", buf);
+    EXPECT_EQ(0, input4.read(buf, maxSize));
+}
+
+TEST(InputTest, ReadMultipleStrings)
+{
+    int count = 3;
+    char* str[] = {"f", "o", "o"};
+    char buf[4] = {'\0', '\0', '\0', '\0'};
+
+    int maxSize = 1;
+    pp::Input input1(count, str, NULL);
+    EXPECT_EQ(1, input1.read(buf, maxSize));
+    EXPECT_EQ('f', buf[0]);
+    EXPECT_EQ(1, input1.read(buf, maxSize));
+    EXPECT_EQ('o', buf[0]);
+    EXPECT_EQ(1, input1.read(buf, maxSize));
+    EXPECT_EQ('o', buf[0]);
+    EXPECT_EQ(0, input1.read(buf, maxSize));
+
+    maxSize = 2;
+    pp::Input input2(count, str, NULL);
+    EXPECT_EQ(2, input2.read(buf, maxSize));
+    EXPECT_STREQ("fo", buf);
+    EXPECT_EQ(1, input2.read(buf, maxSize));
+    EXPECT_EQ('o', buf[0]);
+    EXPECT_EQ(0, input2.read(buf, maxSize));
+
+    maxSize = 3;
+    pp::Input input3(count, str, NULL);
+    EXPECT_EQ(3, input3.read(buf, maxSize));
+    EXPECT_STREQ("foo", buf);
+    EXPECT_EQ(0, input3.read(buf, maxSize));
+
+    maxSize = 4;
+    pp::Input input4(count, str, NULL);
+    EXPECT_EQ(3, input4.read(buf, maxSize));
+    EXPECT_STREQ("foo", buf);
+    EXPECT_EQ(0, input4.read(buf, maxSize));
+}
+
+TEST(InputTest, ReadStringsWithLength)
+{
+    int count = 2;
+    char* str[] = {"foo", "bar"};
+    // Note that the length for the first string is 2 which is less than
+    // strlen(str[0]. We want to make sure that the last character is ignored.
+    int length[] = {2, 3};
+    char buf[6] = {'\0', '\0', '\0', '\0', '\0', '\0'};
+    int maxSize = 5;
+
+    pp::Input input(count, str, length);
+    EXPECT_EQ(maxSize, input.read(buf, maxSize));
+    EXPECT_STREQ("fobar", buf);
+}
+