Preprocessor update from ANGLE

src/OpenGL/compiler/Diagnostics.h

@@ -15,7 +15,7 @@
 #ifndef COMPILER_DIAGNOSTICS_H_
 #define COMPILER_DIAGNOSTICS_H_
 
-#include "preprocessor/Diagnostics.h"
+#include "preprocessor/DiagnosticsBase.h"
 
 class TInfoSink;
 

src/OpenGL/compiler/DirectiveHandler.cpp

@@ -54,7 +54,8 @@ void TDirectiveHandler::handleError(const pp::SourceLocation& loc,
 
 void TDirectiveHandler::handlePragma(const pp::SourceLocation& loc,
                                      const std::string& name,
-                                     const std::string& value)
+                                     const std::string& value,
+                                     bool stdgl)
 {
 	static const std::string kSTDGL("STDGL");
 	static const std::string kOptimize("optimize");
@@ -63,7 +64,7 @@ void TDirectiveHandler::handlePragma(const pp::SourceLocation& loc,
 	static const std::string kOff("off");
 
 	bool invalidValue = false;
-	if (name == kSTDGL)
+	if (stdgl || (name == kSTDGL))
 	{
 		// The STDGL pragma is used to reserve pragmas for use by future
 		// revisions of GLSL. Ignore it.
@@ -83,7 +84,7 @@ void TDirectiveHandler::handlePragma(const pp::SourceLocation& loc,
 	}
 	else
 	{
-		mDiagnostics.report(pp::Diagnostics::UNRECOGNIZED_PRAGMA, loc, name);
+		mDiagnostics.report(pp::Diagnostics::PP_UNRECOGNIZED_PRAGMA, loc, name);
 		return;
 	}
 

src/OpenGL/compiler/DirectiveHandler.h

@@ -17,7 +17,7 @@
 
 #include "ExtensionBehavior.h"
 #include "Pragma.h"
-#include "preprocessor/DirectiveHandler.h"
+#include "preprocessor/DirectiveHandlerBase.h"
 
 class TDiagnostics;
 
@@ -37,7 +37,8 @@ public:
 
 	virtual void handlePragma(const pp::SourceLocation& loc,
 	                          const std::string& name,
-	                          const std::string& value);
+	                          const std::string& value,
+	                          bool stdgl);
 
 	virtual void handleExtension(const pp::SourceLocation& loc,
 	                             const std::string& name,

src/OpenGL/compiler/ParseHelper.cpp

@@ -1122,10 +1122,10 @@ void TParseContext::handleExtensionDirective(const TSourceLoc &line, const char*
 	mDirectiveHandler.handleExtension(loc, extName, behavior);
 }
 
-void TParseContext::handlePragmaDirective(const TSourceLoc &line, const char* name, const char* value)
+void TParseContext::handlePragmaDirective(const TSourceLoc &line, const char* name, const char* value, bool stdgl)
 {
 	pp::SourceLocation loc(line.first_file, line.first_line);
-	mDirectiveHandler.handlePragma(loc, name, value);
+	mDirectiveHandler.handlePragma(loc, name, value, stdgl);
 }
 
 /////////////////////////////////////////////////////////////////////////////////

src/OpenGL/compiler/ParseHelper.h

@@ -56,7 +56,7 @@ public:
 			mDefaultBlockStorage(EbsShared),
 			mDiagnostics(is),
 			mDirectiveHandler(ext, mDiagnostics, mShaderVersion),
-			mPreprocessor(&mDiagnostics, &mDirectiveHandler),
+			mPreprocessor(&mDiagnostics, &mDirectiveHandler, pp::PreprocessorSettings()),
 			mScanner(nullptr),
 			mUsesFragData(false),
 			mUsesFragColor(false) {  }
@@ -148,7 +148,7 @@ public:
 	void handleExtensionDirective(const TSourceLoc &line, const char* extName, const char* behavior);
 
 	const TPragma& pragma() const { return mDirectiveHandler.pragma(); }
-	void handlePragmaDirective(const TSourceLoc &line, const char* name, const char* value);
+	void handlePragmaDirective(const TSourceLoc &line, const char* name, const char* value, bool stdgl);
 
 	bool containsSampler(TType& type);
 	const TFunction* findFunction(const TSourceLoc &line, TFunction* pfnCall, bool *builtIn = 0);

src/OpenGL/compiler/preprocessor/Diagnostics.h

-// Copyright 2016 The SwiftShader Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef COMPILER_PREPROCESSOR_DIAGNOSTICS_H_
-#define COMPILER_PREPROCESSOR_DIAGNOSTICS_H_
-
-#include <string>
-
-namespace pp
-{
-
-struct SourceLocation;
-
-// Base class for reporting diagnostic messages.
-// Derived classes are responsible for formatting and printing the messages.
-class Diagnostics
-{
-public:
-	enum Severity
-	{
-		PP_ERROR,
-		PP_WARNING
-	};
-	enum ID
-	{
-		ERROR_BEGIN,
-		INTERNAL_ERROR,
-		OUT_OF_MEMORY,
-		INVALID_CHARACTER,
-		INVALID_NUMBER,
-		INTEGER_OVERFLOW,
-		FLOAT_OVERFLOW,
-		TOKEN_TOO_LONG,
-		INVALID_EXPRESSION,
-		DIVISION_BY_ZERO,
-		EOF_IN_COMMENT,
-		UNEXPECTED_TOKEN,
-		DIRECTIVE_INVALID_NAME,
-		MACRO_NAME_RESERVED,
-		MACRO_REDEFINED,
-		MACRO_PREDEFINED_REDEFINED,
-		MACRO_PREDEFINED_UNDEFINED,
-		MACRO_UNTERMINATED_INVOCATION,
-		MACRO_TOO_FEW_ARGS,
-		MACRO_TOO_MANY_ARGS,
-		MACRO_DUPLICATE_PARAMETER_NAMES,
-		CONDITIONAL_ENDIF_WITHOUT_IF,
-		CONDITIONAL_ELSE_WITHOUT_IF,
-		CONDITIONAL_ELSE_AFTER_ELSE,
-		CONDITIONAL_ELIF_WITHOUT_IF,
-		CONDITIONAL_ELIF_AFTER_ELSE,
-		CONDITIONAL_UNTERMINATED,
-		CONDITIONAL_UNEXPECTED_TOKEN,
-		INVALID_EXTENSION_NAME,
-		INVALID_EXTENSION_BEHAVIOR,
-		INVALID_EXTENSION_DIRECTIVE,
-		INVALID_VERSION_NUMBER,
-		INVALID_VERSION_DIRECTIVE,
-		VERSION_NOT_FIRST_STATEMENT,
-		INVALID_LINE_NUMBER,
-		INVALID_FILE_NUMBER,
-		INVALID_LINE_DIRECTIVE,
-		UNDEFINED_IDENTIFIER,
-		ERROR_END,
-
-		WARNING_BEGIN,
-		EOF_IN_DIRECTIVE,
-		UNRECOGNIZED_PRAGMA,
-		WARNING_END
-	};
-
-	virtual ~Diagnostics();
-
-	void report(ID id, const SourceLocation& loc, const std::string& text);
-
-protected:
-	Severity severity(ID id);
-	std::string message(ID id);
-
-	virtual void print(ID id,
-	                   const SourceLocation& loc,
-	                   const std::string& text) = 0;
-};
-
-}  // namespace pp
-#endif  // COMPILER_PREPROCESSOR_DIAGNOSTICS_H_

src/OpenGL/compiler/preprocessor/Diagnostics.cpp → src/OpenGL/compiler/preprocessor/DiagnosticsBase.cpp

-// Copyright 2016 The SwiftShader Authors. All Rights Reserved.
+// Copyright 2017 The SwiftShader Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "Diagnostics.h"
+#include "DiagnosticsBase.h"
 
 #include <cassert>
 
@@ -23,116 +23,140 @@ Diagnostics::~Diagnostics()
 {
 }
 
-void Diagnostics::report(ID id,
-                         const SourceLocation& loc,
-                         const std::string& text)
+void Diagnostics::report(ID id, const SourceLocation &loc, const std::string &text)
 {
-	// TODO(alokp): Keep a count of errors and warnings.
 	print(id, loc, text);
 }
 
+bool Diagnostics::isError(ID id)
+{
+	if ((id > PP_ERROR_BEGIN) && (id < PP_ERROR_END))
+		return true;
+
+	if ((id > PP_WARNING_BEGIN) && (id < PP_WARNING_END))
+		return false;
+
+	assert(false);
+	return true;
+}
+
 Diagnostics::Severity Diagnostics::severity(ID id)
 {
-	if ((id > ERROR_BEGIN) && (id < ERROR_END))
+	if((id > PP_ERROR_BEGIN) && (id < PP_ERROR_END))
 		return PP_ERROR;
 
-	if ((id > WARNING_BEGIN) && (id < WARNING_END))
+	if((id > PP_WARNING_BEGIN) && (id < PP_WARNING_END))
 		return PP_WARNING;
 
 	assert(false);
 	return PP_ERROR;
 }
 
-std::string Diagnostics::message(ID id)
+const char *Diagnostics::message(ID id)
 {
 	switch (id)
 	{
 	// Errors begin.
-	case INTERNAL_ERROR:
-		  return "internal error";
-	case OUT_OF_MEMORY:
-		  return "out of memory";
-	case INVALID_CHARACTER:
-		  return "invalid character";
-	case INVALID_NUMBER:
-		  return "invalid number";
-	case INTEGER_OVERFLOW:
-		  return "integer overflow";
-	case FLOAT_OVERFLOW:
-		  return "float overflow";
-	case TOKEN_TOO_LONG:
-		  return "token too long";
-	case INVALID_EXPRESSION:
-		  return "invalid expression";
-	case DIVISION_BY_ZERO:
-		  return "division by zero";
-	case EOF_IN_COMMENT:
-		  return "unexpected end of file found in comment";
-	case UNEXPECTED_TOKEN:
-		  return "unexpected token";
-	case DIRECTIVE_INVALID_NAME:
-		  return "invalid directive name";
-	case MACRO_NAME_RESERVED:
-		  return "macro name is reserved";
-	case MACRO_REDEFINED:
-		  return "macro redefined";
-	case MACRO_PREDEFINED_REDEFINED:
-		  return "predefined macro redefined";
-	case MACRO_PREDEFINED_UNDEFINED:
-		  return "predefined macro undefined";
-	case MACRO_UNTERMINATED_INVOCATION:
-		  return "unterminated macro invocation";
-	case MACRO_TOO_FEW_ARGS:
-		  return "Not enough arguments for macro";
-	case MACRO_TOO_MANY_ARGS:
-		  return "Too many arguments for macro";
-	case MACRO_DUPLICATE_PARAMETER_NAMES:
-		  return "duplicate macro parameter name";
-	case CONDITIONAL_ENDIF_WITHOUT_IF:
-		  return "unexpected #endif found without a matching #if";
-	case CONDITIONAL_ELSE_WITHOUT_IF:
-		  return "unexpected #else found without a matching #if";
-	case CONDITIONAL_ELSE_AFTER_ELSE:
-		  return "unexpected #else found after another #else";
-	case CONDITIONAL_ELIF_WITHOUT_IF:
-		  return "unexpected #elif found without a matching #if";
-	case CONDITIONAL_ELIF_AFTER_ELSE:
-		  return "unexpected #elif found after #else";
-	case CONDITIONAL_UNTERMINATED:
-		  return "unexpected end of file found in conditional block";
-	case INVALID_EXTENSION_NAME:
-		  return "invalid extension name";
-	case INVALID_EXTENSION_BEHAVIOR:
-		  return "invalid extension behavior";
-	case INVALID_EXTENSION_DIRECTIVE:
-		  return "invalid extension directive";
-	case INVALID_VERSION_NUMBER:
-		  return "invalid version number";
-	case INVALID_VERSION_DIRECTIVE:
-		  return "invalid version directive";
-	case VERSION_NOT_FIRST_STATEMENT:
+	case PP_INTERNAL_ERROR:
+		return "internal error";
+	case PP_OUT_OF_MEMORY:
+		return "out of memory";
+	case PP_INVALID_CHARACTER:
+		return "invalid character";
+	case PP_INVALID_NUMBER:
+		return "invalid number";
+	case PP_INTEGER_OVERFLOW:
+		return "integer overflow";
+	case PP_FLOAT_OVERFLOW:
+		return "float overflow";
+	case PP_TOKEN_TOO_LONG:
+		return "token too long";
+	case PP_INVALID_EXPRESSION:
+		return "invalid expression";
+	case PP_DIVISION_BY_ZERO:
+		return "division by zero";
+	case PP_EOF_IN_COMMENT:
+		return "unexpected end of file found in comment";
+	case PP_UNEXPECTED_TOKEN:
+		return "unexpected token";
+	case PP_DIRECTIVE_INVALID_NAME:
+		return "invalid directive name";
+	case PP_MACRO_NAME_RESERVED:
+		return "macro name is reserved";
+	case PP_MACRO_REDEFINED:
+		return "macro redefined";
+	case PP_MACRO_PREDEFINED_REDEFINED:
+		return "predefined macro redefined";
+	case PP_MACRO_PREDEFINED_UNDEFINED:
+		return "predefined macro undefined";
+	case PP_MACRO_UNTERMINATED_INVOCATION:
+		return "unterminated macro invocation";
+	case PP_MACRO_UNDEFINED_WHILE_INVOKED:
+		return "macro undefined while being invoked";
+	case PP_MACRO_TOO_FEW_ARGS:
+		return "Not enough arguments for macro";
+	case PP_MACRO_TOO_MANY_ARGS:
+		return "Too many arguments for macro";
+	case PP_MACRO_DUPLICATE_PARAMETER_NAMES:
+		return "duplicate macro parameter name";
+	case PP_MACRO_INVOCATION_CHAIN_TOO_DEEP:
+		return "macro invocation chain too deep";
+	case PP_CONDITIONAL_ENDIF_WITHOUT_IF:
+		return "unexpected #endif found without a matching #if";
+	case PP_CONDITIONAL_ELSE_WITHOUT_IF:
+		return "unexpected #else found without a matching #if";
+	case PP_CONDITIONAL_ELSE_AFTER_ELSE:
+		return "unexpected #else found after another #else";
+	case PP_CONDITIONAL_ELIF_WITHOUT_IF:
+		return "unexpected #elif found without a matching #if";
+	case PP_CONDITIONAL_ELIF_AFTER_ELSE:
+		return "unexpected #elif found after #else";
+	case PP_CONDITIONAL_UNTERMINATED:
+		return "unexpected end of file found in conditional block";
+	case PP_INVALID_EXTENSION_NAME:
+		return "invalid extension name";
+	case PP_INVALID_EXTENSION_BEHAVIOR:
+		return "invalid extension behavior";
+	case PP_INVALID_EXTENSION_DIRECTIVE:
+		return "invalid extension directive";
+	case PP_INVALID_VERSION_NUMBER:
+		return "invalid version number";
+	case PP_INVALID_VERSION_DIRECTIVE:
+		return "invalid version directive";
+	case PP_VERSION_NOT_FIRST_STATEMENT:
 		return "#version directive must occur before anything else, "
 		       "except for comments and white space";
-	case INVALID_LINE_NUMBER:
-		  return "invalid line number";
-	case INVALID_FILE_NUMBER:
-		  return "invalid file number";
-	case INVALID_LINE_DIRECTIVE:
-		  return "invalid line directive";
-	case UNDEFINED_IDENTIFIER:
-		  return "undefined identifier";
+	case PP_VERSION_NOT_FIRST_LINE_ESSL3:
+		return "#version directive must occur on the first line of the shader";
+	case PP_INVALID_LINE_NUMBER:
+		return "invalid line number";
+	case PP_INVALID_FILE_NUMBER:
+		return "invalid file number";
+	case PP_INVALID_LINE_DIRECTIVE:
+		return "invalid line directive";
+	case PP_NON_PP_TOKEN_BEFORE_EXTENSION_ESSL3:
+		return "extension directive must occur before any non-preprocessor tokens in ESSL3";
+	case PP_UNDEFINED_SHIFT:
+		return "shift exponent is negative or undefined";
+	case PP_TOKENIZER_ERROR:
+		return "internal tokenizer error";
 	// Errors end.
 	// Warnings begin.
-	case EOF_IN_DIRECTIVE:
-		  return "unexpected end of file found in directive";
-	case CONDITIONAL_UNEXPECTED_TOKEN:
-		  return "unexpected token after conditional expression";
-	case UNRECOGNIZED_PRAGMA:
-		  return "unrecognized pragma";
+	case PP_EOF_IN_DIRECTIVE:
+		return "unexpected end of file found in directive";
+	case PP_CONDITIONAL_UNEXPECTED_TOKEN:
+		return "unexpected token after conditional expression";
+	case PP_UNRECOGNIZED_PRAGMA:
+		return "unrecognized pragma";
+	case PP_NON_PP_TOKEN_BEFORE_EXTENSION_ESSL1:
+		return "extension directive should occur before any non-preprocessor tokens";
+	case PP_WARNING_MACRO_NAME_RESERVED:
+		return "macro name with a double underscore is reserved - unintented behavior is "
+		       "possible";
 	// Warnings end.
 	default:
-		  assert(false);
-		  return "";
+		assert(false);
+		return "";
 	}
 }
 

src/OpenGL/compiler/preprocessor/DiagnosticsBase.h

+// Copyright 2017 The SwiftShader Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef COMPILER_PREPROCESSOR_DIAGNOSTICSBASE_H_
+#define COMPILER_PREPROCESSOR_DIAGNOSTICSBASE_H_
+
+#include <string>
+
+namespace pp
+{
+
+struct SourceLocation;
+
+// Base class for reporting diagnostic messages.
+// Derived classes are responsible for formatting and printing the messages.
+class Diagnostics
+{
+public:
+	// Severity is used to classify info log messages.
+	enum Severity
+	{
+		PP_WARNING,
+		PP_ERROR
+	};
+
+	enum ID
+	{
+		PP_ERROR_BEGIN,
+		PP_INTERNAL_ERROR,
+		PP_OUT_OF_MEMORY,
+		PP_INVALID_CHARACTER,
+		PP_INVALID_NUMBER,
+		PP_INTEGER_OVERFLOW,
+		PP_FLOAT_OVERFLOW,
+		PP_TOKEN_TOO_LONG,
+		PP_INVALID_EXPRESSION,
+		PP_DIVISION_BY_ZERO,
+		PP_EOF_IN_COMMENT,
+		PP_UNEXPECTED_TOKEN,
+		PP_DIRECTIVE_INVALID_NAME,
+		PP_MACRO_NAME_RESERVED,
+		PP_MACRO_REDEFINED,
+		PP_MACRO_PREDEFINED_REDEFINED,
+		PP_MACRO_PREDEFINED_UNDEFINED,
+		PP_MACRO_UNTERMINATED_INVOCATION,
+		PP_MACRO_UNDEFINED_WHILE_INVOKED,
+		PP_MACRO_TOO_FEW_ARGS,
+		PP_MACRO_TOO_MANY_ARGS,
+		PP_MACRO_DUPLICATE_PARAMETER_NAMES,
+		PP_MACRO_INVOCATION_CHAIN_TOO_DEEP,
+		PP_CONDITIONAL_ENDIF_WITHOUT_IF,
+		PP_CONDITIONAL_ELSE_WITHOUT_IF,
+		PP_CONDITIONAL_ELSE_AFTER_ELSE,
+		PP_CONDITIONAL_ELIF_WITHOUT_IF,
+		PP_CONDITIONAL_ELIF_AFTER_ELSE,
+		PP_CONDITIONAL_UNTERMINATED,
+		PP_CONDITIONAL_UNEXPECTED_TOKEN,
+		PP_INVALID_EXTENSION_NAME,
+		PP_INVALID_EXTENSION_BEHAVIOR,
+		PP_INVALID_EXTENSION_DIRECTIVE,
+		PP_INVALID_VERSION_NUMBER,
+		PP_INVALID_VERSION_DIRECTIVE,
+		PP_VERSION_NOT_FIRST_STATEMENT,
+		PP_VERSION_NOT_FIRST_LINE_ESSL3,
+		PP_INVALID_LINE_NUMBER,
+		PP_INVALID_FILE_NUMBER,
+		PP_INVALID_LINE_DIRECTIVE,
+		PP_NON_PP_TOKEN_BEFORE_EXTENSION_ESSL3,
+		PP_UNDEFINED_SHIFT,
+		PP_TOKENIZER_ERROR,
+		PP_ERROR_END,
+
+		PP_WARNING_BEGIN,
+		PP_EOF_IN_DIRECTIVE,
+		PP_UNRECOGNIZED_PRAGMA,
+		PP_NON_PP_TOKEN_BEFORE_EXTENSION_ESSL1,
+		PP_WARNING_MACRO_NAME_RESERVED,
+		PP_WARNING_END
+	};
+
+	virtual ~Diagnostics();
+
+	void report(ID id, const SourceLocation &loc, const std::string &text);
+
+protected:
+	bool isError(ID id);
+	const char *message(ID id);
+	Severity severity(ID id);
+
+	virtual void print(ID id, const SourceLocation &loc, const std::string &text) = 0;
+};
+
+}  // namespace pp
+
+#endif  // COMPILER_PREPROCESSOR_DIAGNOSTICSBASE_H_

src/OpenGL/compiler/preprocessor/DirectiveHandler.cpp → src/OpenGL/compiler/preprocessor/DirectiveHandlerBase.cpp

-// Copyright 2016 The SwiftShader Authors. All Rights Reserved.
+// Copyright 2017 The SwiftShader Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "DirectiveHandler.h"
+#include "DirectiveHandlerBase.h"
 
 namespace pp
 {

src/OpenGL/compiler/preprocessor/DirectiveHandler.h → src/OpenGL/compiler/preprocessor/DirectiveHandlerBase.h

-// Copyright 2016 The SwiftShader Authors. All Rights Reserved.
+// Copyright 2017 The SwiftShader Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef COMPILER_PREPROCESSOR_DIRECTIVE_HANDLER_H_
-#define COMPILER_PREPROCESSOR_DIRECTIVE_HANDLER_H_
+#ifndef COMPILER_PREPROCESSOR_DIRECTIVEHANDLERBASE_H_
+#define COMPILER_PREPROCESSOR_DIRECTIVEHANDLERBASE_H_
 
 #include <string>
 
@@ -28,24 +28,24 @@ struct SourceLocation;
 // handling them in an appropriate manner.
 class DirectiveHandler
 {
-public:
+  public:
 	virtual ~DirectiveHandler();
 
-	virtual void handleError(const SourceLocation& loc,
-	                         const std::string& msg) = 0;
+	virtual void handleError(const SourceLocation &loc, const std::string &msg) = 0;
 
 	// Handle pragma of form: #pragma name[(value)]
-	virtual void handlePragma(const SourceLocation& loc,
-	                          const std::string& name,
-	                          const std::string& value) = 0;
+	virtual void handlePragma(const SourceLocation &loc,
+	                          const std::string &name,
+	                          const std::string &value,
+	                          bool stdgl) = 0;
 
-	virtual void handleExtension(const SourceLocation& loc,
-	                             const std::string& name,
-	                             const std::string& behavior) = 0;
+	virtual void handleExtension(const SourceLocation &loc,
+	                             const std::string &name,
+	                             const std::string &behavior) = 0;
 
-	virtual void handleVersion(const SourceLocation& loc,
-	                           int version) = 0;
+	virtual void handleVersion(const SourceLocation &loc, int version) = 0;
 };
 
 }  // namespace pp
-#endif  // COMPILER_PREPROCESSOR_DIRECTIVE_HANDLER_H_
+
+#endif  // COMPILER_PREPROCESSOR_DIRECTIVEHANDLERBASE_H_

src/OpenGL/compiler/preprocessor/DirectiveParser.h

@@ -30,35 +30,37 @@ class Tokenizer;
 class DirectiveParser : public Lexer
 {
 public:
-	DirectiveParser(Tokenizer* tokenizer,
-	                MacroSet* macroSet,
-	                Diagnostics* diagnostics,
-	                DirectiveHandler* directiveHandler);
+	DirectiveParser(Tokenizer *tokenizer,
+	                MacroSet *macroSet,
+	                Diagnostics *diagnostics,
+	                DirectiveHandler *directiveHandler,
+	                int maxMacroExpansionDepth);
+	~DirectiveParser() override;
 
-	virtual void lex(Token* token);
+	void lex(Token *token) override;
 
 private:
 	PP_DISALLOW_COPY_AND_ASSIGN(DirectiveParser);
 
-	void parseDirective(Token* token);
-	void parseDefine(Token* token);
-	void parseUndef(Token* token);
-	void parseIf(Token* token);
-	void parseIfdef(Token* token);
-	void parseIfndef(Token* token);
-	void parseElse(Token* token);
-	void parseElif(Token* token);
-	void parseEndif(Token* token);
-	void parseError(Token* token);
-	void parsePragma(Token* token);
-	void parseExtension(Token* token);
-	void parseVersion(Token* token);
-	void parseLine(Token* token);
+	void parseDirective(Token *token);
+	void parseDefine(Token *token);
+	void parseUndef(Token *token);
+	void parseIf(Token *token);
+	void parseIfdef(Token *token);
+	void parseIfndef(Token *token);
+	void parseElse(Token *token);
+	void parseElif(Token *token);
+	void parseEndif(Token *token);
+	void parseError(Token *token);
+	void parsePragma(Token *token);
+	void parseExtension(Token *token);
+	void parseVersion(Token *token);
+	void parseLine(Token *token);
 
 	bool skipping() const;
-	void parseConditionalIf(Token* token);
-	int parseExpressionIf(Token* token);
-	int parseExpressionIfdef(Token* token);
+	void parseConditionalIf(Token *token);
+	int parseExpressionIf(Token *token);
+	int parseExpressionIfdef(Token *token);
 
 	struct ConditionalBlock
 	{
@@ -78,11 +80,16 @@ private:
 		}
 	};
 	bool mPastFirstStatement;
+	bool mSeenNonPreprocessorToken;  // Tracks if a non-preprocessor token has been seen yet.  Some
+	                                 // macros, such as
+	                                 // #extension must be declared before all shader code.
 	std::vector<ConditionalBlock> mConditionalStack;
-	Tokenizer* mTokenizer;
-	MacroSet* mMacroSet;
-	Diagnostics* mDiagnostics;
-	DirectiveHandler* mDirectiveHandler;
+	Tokenizer *mTokenizer;
+	MacroSet *mMacroSet;
+	Diagnostics *mDiagnostics;
+	DirectiveHandler *mDirectiveHandler;
+	int mShaderVersion;
+	int mMaxMacroExpansionDepth;
 };
 
 }  // namespace pp

src/OpenGL/compiler/preprocessor/ExpressionParser.h

@@ -15,27 +15,34 @@
 #ifndef COMPILER_PREPROCESSOR_EXPRESSION_PARSER_H_
 #define COMPILER_PREPROCESSOR_EXPRESSION_PARSER_H_
 
-#include "pp_utils.h"
+#include "DiagnosticsBase.h"
 
 namespace pp
 {
 
-class Diagnostics;
 class Lexer;
 struct Token;
 
 class ExpressionParser
 {
 public:
-	ExpressionParser(Lexer* lexer, Diagnostics* diagnostics);
+	struct ErrorSettings
+	{
+		Diagnostics::ID unexpectedIdentifier;
+		bool integerLiteralsMustFit32BitSignedRange;
+	};
 
-	bool parse(Token* token, int* result);
+	ExpressionParser(Lexer *lexer, Diagnostics *diagnostics);
 
-private:
-	PP_DISALLOW_COPY_AND_ASSIGN(ExpressionParser);
+	bool parse(Token *token,
+	           int *result,
+	           bool parsePresetToken,
+	           const ErrorSettings &errorSettings,
+	           bool *valid);
 
-	Lexer* mLexer;
-	Diagnostics* mDiagnostics;
+private:
+	Lexer *mLexer;
+	Diagnostics *mDiagnostics;
 };
 
 }  // namespace pp

src/OpenGL/compiler/preprocessor/Input.cpp

@@ -25,27 +25,101 @@ Input::Input() : mCount(0), mString(0)
 {
 }
 
-Input::Input(int count, const char* const string[], const int length[]) :
-	mCount(count),
-	mString(string)
+Input::~Input()
+{
+}
+
+Input::Input(size_t count, const char *const string[], const int length[])
+	: mCount(count), mString(string)
 {
-	assert(mCount >= 0);
 	mLength.reserve(mCount);
-	for (int i = 0; i < mCount; ++i)
+	for (size_t i = 0; i < mCount; ++i)
 	{
 		int len = length ? length[i] : -1;
-		mLength.push_back(len < 0 ? strlen(mString[i]) : len);
+		mLength.push_back(len < 0 ? std::strlen(mString[i]) : len);
 	}
 }
 
-int Input::read(char* buf, int maxSize)
+const char *Input::skipChar()
 {
-	int nRead = 0;
-	while ((nRead < maxSize) && (mReadLoc.sIndex < mCount))
+	// This function should only be called when there is a character to skip.
+	assert(mReadLoc.cIndex < mLength[mReadLoc.sIndex]);
+	++mReadLoc.cIndex;
+	if (mReadLoc.cIndex == mLength[mReadLoc.sIndex])
+	{
+		++mReadLoc.sIndex;
+		mReadLoc.cIndex = 0;
+	}
+	if (mReadLoc.sIndex >= mCount)
 	{
-		int size = mLength[mReadLoc.sIndex] - mReadLoc.cIndex;
-		size = std::min(size, maxSize);
-		memcpy(buf + nRead, mString[mReadLoc.sIndex] + mReadLoc.cIndex, size);
+		return nullptr;
+	}
+	return mString[mReadLoc.sIndex] + mReadLoc.cIndex;
+}
+
+size_t Input::read(char *buf, size_t maxSize, int *lineNo)
+{
+	size_t nRead = 0;
+	// The previous call to read might have stopped copying the string when encountering a line
+	// continuation. Check for this possibility first.
+	if (mReadLoc.sIndex < mCount && maxSize > 0)
+	{
+		const char *c = mString[mReadLoc.sIndex] + mReadLoc.cIndex;
+		if ((*c) == '\\')
+		{
+			c = skipChar();
+			if (c != nullptr && (*c) == '\n')
+			{
+				// Line continuation of backslash + newline.
+				skipChar();
+				// Fake an EOF if the line number would overflow.
+				if (*lineNo == INT_MAX)
+				{
+					return 0;
+				}
+				++(*lineNo);
+			}
+			else if (c != nullptr && (*c) == '\r')
+			{
+				// Line continuation. Could be backslash + '\r\n' or just backslash + '\r'.
+				c = skipChar();
+				if (c != nullptr && (*c) == '\n')
+				{
+					skipChar();
+				}
+				// Fake an EOF if the line number would overflow.
+				if (*lineNo == INT_MAX)
+				{
+					return 0;
+				}
+				++(*lineNo);
+			}
+			else
+			{
+				// Not line continuation, so write the skipped backslash to buf.
+				*buf = '\\';
+				++nRead;
+			}
+		}
+	}
+
+	size_t maxRead = maxSize;
+	while ((nRead < maxRead) && (mReadLoc.sIndex < mCount))
+	{
+		size_t size = mLength[mReadLoc.sIndex] - mReadLoc.cIndex;
+		size		= std::min(size, maxSize);
+		for (size_t i = 0; i < size; ++i)
+		{
+			// Stop if a possible line continuation is encountered.
+			// It will be processed on the next call on input, which skips it
+			// and increments line number if necessary.
+			if (*(mString[mReadLoc.sIndex] + mReadLoc.cIndex + i) == '\\')
+			{
+				size	= i;
+				maxRead = nRead + size;  // Stop reading right before the backslash.
+			}
+		}
+		std::memcpy(buf + nRead, mString[mReadLoc.sIndex] + mReadLoc.cIndex, size);
 		nRead += size;
 		mReadLoc.cIndex += size;
 

src/OpenGL/compiler/preprocessor/Input.h

@@ -15,6 +15,7 @@
 #ifndef COMPILER_PREPROCESSOR_INPUT_H_
 #define COMPILER_PREPROCESSOR_INPUT_H_
 
+#include <cstddef>
 #include <vector>
 
 namespace pp
@@ -25,28 +26,33 @@ class Input
 {
 public:
 	Input();
-	Input(int count, const char* const string[], const int length[]);
+	~Input();
+	Input(size_t count, const char *const string[], const int length[]);
 
-	int count() const { return mCount; }
-	const char* string(int index) const { return mString[index]; }
-	int length(int index) const { return mLength[index]; }
+	size_t count() const { return mCount; }
+	const char *string(size_t index) const { return mString[index]; }
+	size_t length(size_t index) const { return mLength[index]; }
 
-	int read(char* buf, int maxSize);
+	size_t read(char *buf, size_t maxSize, int *lineNo);
 
 	struct Location
 	{
-		int sIndex;  // String index;
-		int cIndex;  // Char index.
+		size_t sIndex;  // String index;
+		size_t cIndex;  // Char index.
 
-		Location() : sIndex(0), cIndex(0) { }
+		Location() : sIndex(0), cIndex(0) {}
 	};
-	const Location& readLoc() const { return mReadLoc; }
+	const Location &readLoc() const { return mReadLoc; }
 
 private:
+	// Skip a character and return the next character after the one that was skipped.
+	// Return nullptr if data runs out.
+	const char *skipChar();
+
 	// Input.
-	int mCount;
-	const char* const* mString;
-	std::vector<int> mLength;
+	size_t mCount;
+	const char *const *mString;
+	std::vector<size_t> mLength;
 
 	Location mReadLoc;
 };

src/OpenGL/compiler/preprocessor/Lexer.h

@@ -25,7 +25,7 @@ class Lexer
 public:
 	virtual ~Lexer();
 
-	virtual void lex(Token* token) = 0;
+	virtual void lex(Token *token) = 0;
 };
 
 }  // namespace pp

src/OpenGL/compiler/preprocessor/Macro.cpp

@@ -19,12 +19,33 @@
 namespace pp
 {
 
-bool Macro::equals(const Macro& other) const
+Macro::Macro() : predefined(false), disabled(false), expansionCount(0), type(kTypeObj)
 {
-    return (type == other.type) &&
-           (name == other.name) &&
-           (parameters == other.parameters) &&
-           (replacements == other.replacements);
+}
+
+Macro::~Macro()
+{
+}
+
+bool Macro::equals(const Macro &other) const
+{
+	return (type == other.type) && (name == other.name) && (parameters == other.parameters) &&
+	       (replacements == other.replacements);
+}
+
+void PredefineMacro(MacroSet *macroSet, const char *name, int value)
+{
+	Token token;
+	token.type = Token::CONST_INT;
+	token.text = std::to_string(value);
+
+	std::shared_ptr<Macro> macro = std::make_shared<Macro>();
+	macro->predefined = true;
+	macro->type = Macro::kTypeObj;
+	macro->name = name;
+	macro->replacements.push_back(token);
+
+	(*macroSet)[name] = macro;
 }
 
 }  // namespace pp

src/OpenGL/compiler/preprocessor/Macro.h

@@ -16,6 +16,7 @@
 #define COMPILER_PREPROCESSOR_MACRO_H_
 
 #include <map>
+#include <memory>
 #include <string>
 #include <vector>
 
@@ -34,11 +35,13 @@ struct Macro
     typedef std::vector<std::string> Parameters;
     typedef std::vector<Token> Replacements;
 
-    Macro() : predefined(false), disabled(false), type(kTypeObj) { }
-    bool equals(const Macro& other) const;
+    Macro();
+    ~Macro();
+    bool equals(const Macro &other) const;
 
     bool predefined;
     mutable bool disabled;
+    mutable int expansionCount;
 
     Type type;
     std::string name;
@@ -46,7 +49,9 @@ struct Macro
     Replacements replacements;
 };
 
-typedef std::map<std::string, Macro> MacroSet;
+typedef std::map<std::string, std::shared_ptr<Macro>> MacroSet;
+
+void PredefineMacro(MacroSet *macroSet, const char *name, int value);
 
 }  // namespace pp
 #endif  // COMPILER_PREPROCESSOR_MACRO_H_

src/OpenGL/compiler/preprocessor/MacroExpander.h

@@ -15,7 +15,6 @@
 #ifndef COMPILER_PREPROCESSOR_MACRO_EXPANDER_H_
 #define COMPILER_PREPROCESSOR_MACRO_EXPANDER_H_
 
-#include <cassert>
 #include <memory>
 #include <vector>
 
@@ -27,58 +26,67 @@ namespace pp
 {
 
 class Diagnostics;
+struct SourceLocation;
 
 class MacroExpander : public Lexer
 {
 public:
-	MacroExpander(Lexer* lexer, MacroSet* macroSet, Diagnostics* diagnostics, bool parseDefined);
-	virtual ~MacroExpander();
+	MacroExpander(Lexer *lexer, MacroSet *macroSet, Diagnostics *diagnostics, bool parseDefined, int allowedMacroExpansionDepth);
+	~MacroExpander() override;
 
-	virtual void lex(Token* token);
+	void lex(Token *token) override;
 
 private:
 	PP_DISALLOW_COPY_AND_ASSIGN(MacroExpander);
 
-	void getToken(Token* token);
-	void ungetToken(const Token& token);
+	void getToken(Token *token);
+	void ungetToken(const Token &token);
 	bool isNextTokenLeftParen();
 
-	bool pushMacro(const Macro& macro, const Token& identifier);
+	bool pushMacro(std::shared_ptr<Macro> macro, const Token &identifier);
 	void popMacro();
 
-	bool expandMacro(const Macro& macro,
-	                 const Token& identifier,
-	                 std::vector<Token>* replacements);
+	bool expandMacro(const Macro &macro, const Token &identifier, std::vector<Token> *replacements);
 
 	typedef std::vector<Token> MacroArg;
-	bool collectMacroArgs(const Macro& macro,
-	                      const Token& identifier,
-	                      std::vector<MacroArg>* args);
-	void replaceMacroParams(const Macro& macro,
-	                        const std::vector<MacroArg>& args,
-	                        std::vector<Token>* replacements);
+	bool collectMacroArgs(const Macro &macro,
+	                      const Token &identifier,
+	                      std::vector<MacroArg> *args,
+	                      SourceLocation *closingParenthesisLocation);
+	void replaceMacroParams(const Macro &macro,
+	                        const std::vector<MacroArg> &args,
+	                        std::vector<Token> *replacements);
 
 	struct MacroContext
 	{
-		const Macro* macro;
-		size_t index;
+		MacroContext();
+		~MacroContext();
+		bool empty() const;
+		const Token &get();
+		void unget();
+
+		std::shared_ptr<Macro> macro;
+		std::size_t index;
 		std::vector<Token> replacements;
-
-		MacroContext() : macro(0), index(0) { }
-		bool empty() const { return index == replacements.size(); }
-		const Token& get() { return replacements[index++]; }
-		void unget() { assert(index > 0); --index; }
 	};
 
-	Lexer* mLexer;
-	MacroSet* mMacroSet;
-	Diagnostics* mDiagnostics;
+	Lexer *mLexer;
+	MacroSet *mMacroSet;
+	Diagnostics *mDiagnostics;
 	const bool mParseDefined;
 
-	Token* mReserveToken;
-	std::vector<MacroContext*> mContextStack;
+	std::unique_ptr<Token> mReserveToken;
+	std::vector<MacroContext *> mContextStack;
+	size_t mTotalTokensInContexts;
+
+	int mAllowedMacroExpansionDepth;
+
+	bool mDeferReenablingMacros;
+	std::vector<std::shared_ptr<Macro>> mMacrosToReenable;
+
+	class ScopedMacroReenabler;
 };
 
 }  // namespace pp
-#endif  // COMPILER_PREPROCESSOR_MACRO_EXPANDER_H_
+#endif  // COMPILER_PREPROCESSOR_MACROEXPANDER_H_
 

src/OpenGL/compiler/preprocessor/Preprocessor.cpp

@@ -15,9 +15,8 @@
 #include "Preprocessor.h"
 
 #include <cassert>
-#include <sstream>
 
-#include "Diagnostics.h"
+#include "DiagnosticsBase.h"
 #include "DirectiveParser.h"
 #include "Macro.h"
 #include "MacroExpander.h"
@@ -29,25 +28,28 @@ namespace pp
 
 struct PreprocessorImpl
 {
-	Diagnostics* diagnostics;
+	Diagnostics *diagnostics;
 	MacroSet macroSet;
 	Tokenizer tokenizer;
 	DirectiveParser directiveParser;
 	MacroExpander macroExpander;
 
-	PreprocessorImpl(Diagnostics* diag, DirectiveHandler* directiveHandler) :
-		diagnostics(diag),
-		tokenizer(diag),
-		directiveParser(&tokenizer, &macroSet, diag, directiveHandler),
-		macroExpander(&directiveParser, &macroSet, diag, false)
+	PreprocessorImpl(Diagnostics *diag,
+	                 DirectiveHandler *directiveHandler,
+	                 const PreprocessorSettings &settings)
+	    : diagnostics(diag),
+	      tokenizer(diag),
+	      directiveParser(&tokenizer, &macroSet, diag, directiveHandler,  settings.maxMacroExpansionDepth),
+	      macroExpander(&directiveParser, &macroSet, diag, false, settings.maxMacroExpansionDepth)
 	{
 	}
 };
 
-Preprocessor::Preprocessor(Diagnostics* diagnostics,
-                           DirectiveHandler* directiveHandler)
+Preprocessor::Preprocessor(Diagnostics *diagnostics,
+                           DirectiveHandler *directiveHandler,
+                           const PreprocessorSettings &settings)
 {
-	mImpl = new PreprocessorImpl(diagnostics, directiveHandler);
+	mImpl = new PreprocessorImpl(diagnostics, directiveHandler, settings);
 }
 
 Preprocessor::~Preprocessor()
@@ -55,40 +57,25 @@ Preprocessor::~Preprocessor()
 	delete mImpl;
 }
 
-bool Preprocessor::init(int count,
-                        const char* const string[],
-                        const int length[])
+bool Preprocessor::init(size_t count, const char *const string[], const int length[])
 {
-	static const int kGLSLVersion = 100;
+	static const int kDefaultGLSLVersion = 100;
 
 	// Add standard pre-defined macros.
 	predefineMacro("__LINE__", 0);
 	predefineMacro("__FILE__", 0);
-	predefineMacro("__VERSION__", kGLSLVersion);
+	predefineMacro("__VERSION__", kDefaultGLSLVersion);
 	predefineMacro("GL_ES", 1);
 
 	return mImpl->tokenizer.init(count, string, length);
 }
 
-void Preprocessor::predefineMacro(const char* name, int value)
+void Preprocessor::predefineMacro(const char *name, int value)
 {
-	std::ostringstream stream;
-	stream << value;
-
-	Token token;
-	token.type = Token::CONST_INT;
-	token.text = stream.str();
-
-	Macro macro;
-	macro.predefined = true;
-	macro.type = Macro::kTypeObj;
-	macro.name = name;
-	macro.replacements.push_back(token);
-
-	mImpl->macroSet[name] = macro;
+	PredefineMacro(&mImpl->macroSet, name, value);
 }
 
-void Preprocessor::lex(Token* token)
+void Preprocessor::lex(Token *token)
 {
 	bool validToken = false;
 	while (!validToken)
@@ -102,40 +89,12 @@ void Preprocessor::lex(Token* token)
 		case Token::PP_HASH:
 			assert(false);
 			break;
-		case Token::CONST_INT:
-		  {
-			int val = 0;
-			if (!token->iValue(&val))
-			{
-				// Do not mark the token as invalid.
-				// Just emit the diagnostic and reset value to 0.
-				mImpl->diagnostics->report(Diagnostics::INTEGER_OVERFLOW,
-				                           token->location, token->text);
-				token->text.assign("0");
-			}
-			validToken = true;
-			break;
-		  }
-		case Token::CONST_FLOAT:
-		  {
-			float val = 0;
-			if (!token->fValue(&val))
-			{
-				// Do not mark the token as invalid.
-				// Just emit the diagnostic and reset value to 0.0.
-				mImpl->diagnostics->report(Diagnostics::FLOAT_OVERFLOW,
-				                           token->location, token->text);
-				token->text.assign("0.0");
-			}
-			validToken = true;
-			break;
-		  }
 		case Token::PP_NUMBER:
-			mImpl->diagnostics->report(Diagnostics::INVALID_NUMBER,
+			mImpl->diagnostics->report(Diagnostics::PP_INVALID_NUMBER,
 			                           token->location, token->text);
 			break;
 		case Token::PP_OTHER:
-			mImpl->diagnostics->report(Diagnostics::INVALID_CHARACTER,
+			mImpl->diagnostics->report(Diagnostics::PP_INVALID_CHARACTER,
 			                           token->location, token->text);
 			break;
 		default:
@@ -145,5 +104,9 @@ void Preprocessor::lex(Token* token)
 	}
 }
 
-}  // namespace pp
+void Preprocessor::setMaxTokenSize(size_t maxTokenSize)
+{
+	mImpl->tokenizer.setMaxTokenSize(maxTokenSize);
+}
 
+}  // namespace pp

src/OpenGL/compiler/preprocessor/Preprocessor.h

@@ -25,10 +25,18 @@ class DirectiveHandler;
 struct PreprocessorImpl;
 struct Token;
 
+struct PreprocessorSettings
+{
+	PreprocessorSettings() : maxMacroExpansionDepth(1000) {}
+	int maxMacroExpansionDepth;
+};
+
 class Preprocessor
 {
 public:
-	Preprocessor(Diagnostics* diagnostics, DirectiveHandler* directiveHandler);
+	Preprocessor(Diagnostics *diagnostics,
+	             DirectiveHandler *directiveHandler,
+	             const PreprocessorSettings &settings);
 	~Preprocessor();
 
 	// count: specifies the number of elements in the string and length arrays.
@@ -40,16 +48,19 @@ public:
 	// Each element in the length array may contain the length of the
 	// corresponding string or a value less than 0 to indicate that the string
 	// is null terminated.
-	bool init(int count, const char* const string[], const int length[]);
+	bool init(size_t count, const char *const string[], const int length[]);
 	// Adds a pre-defined macro.
-	void predefineMacro(const char* name, int value);
+	void predefineMacro(const char *name, int value);
+
+	void lex(Token *token);
 
-	void lex(Token* token);
+	// Set maximum preprocessor token size
+	void setMaxTokenSize(size_t maxTokenSize);
 
 private:
 	PP_DISALLOW_COPY_AND_ASSIGN(Preprocessor);
 
-	PreprocessorImpl* mImpl;
+	PreprocessorImpl *mImpl;
 };
 
 }  // namespace pp

src/OpenGL/compiler/preprocessor/SourceLocation.h

@@ -20,10 +20,10 @@ namespace pp
 
 struct SourceLocation
 {
-	SourceLocation() : file(0), line(0) { }
-	SourceLocation(int f, int l) : file(f), line(l) { }
+	SourceLocation() : file(0), line(0) {}
+	SourceLocation(int f, int l) : file(f), line(l) {}
 
-	bool equals(const SourceLocation& other) const
+	bool equals(const SourceLocation &other) const
 	{
 		return (file == other.file) && (line == other.line);
 	}
@@ -32,12 +32,12 @@ struct SourceLocation
 	int line;
 };
 
-inline bool operator==(const SourceLocation& lhs, const SourceLocation& rhs)
+inline bool operator==(const SourceLocation &lhs, const SourceLocation &rhs)
 {
 	return lhs.equals(rhs);
 }
 
-inline bool operator!=(const SourceLocation& lhs, const SourceLocation& rhs)
+inline bool operator!=(const SourceLocation &lhs, const SourceLocation &rhs)
 {
 	return !lhs.equals(rhs);
 }

src/OpenGL/compiler/preprocessor/Token.cpp

@@ -29,7 +29,7 @@ void Token::reset()
 	text.clear();
 }
 
-bool Token::equals(const Token& other) const
+bool Token::equals(const Token &other) const
 {
 	return (type == other.type) &&
 	       (flags == other.flags) &&
@@ -61,25 +61,25 @@ void Token::setExpansionDisabled(bool disable)
 		flags &= ~EXPANSION_DISABLED;
 }
 
-bool Token::iValue(int* value) const
+bool Token::iValue(int *value) const
 {
 	assert(type == CONST_INT);
 	return numeric_lex_int(text, value);
 }
 
-bool Token::uValue(unsigned int* value) const
+bool Token::uValue(unsigned int *value) const
 {
 	assert(type == CONST_INT);
 	return numeric_lex_int(text, value);
 }
 
-bool Token::fValue(float* value) const
+bool Token::fValue(float *value) const
 {
 	assert(type == CONST_FLOAT);
 	return numeric_lex_float(text, value);
 }
 
-std::ostream& operator<<(std::ostream& out, const Token& token)
+std::ostream &operator<<(std::ostream &out, const Token &token)
 {
 	if (token.hasLeadingSpace())
 		out << " ";

src/OpenGL/compiler/preprocessor/Token.h

@@ -27,6 +27,8 @@ struct Token
 {
 	enum Type
 	{
+		// Calling this ERROR causes a conflict with wingdi.h
+		GOT_ERROR = -1,
 		LAST = 0,  // EOF.
 
 		IDENTIFIER = 258,
@@ -70,10 +72,10 @@ struct Token
 		EXPANSION_DISABLED = 1 << 2
 	};
 
-	Token() : type(0), flags(0) { }
+	Token() : type(0), flags(0) {}
 
 	void reset();
-	bool equals(const Token& other) const;
+	bool equals(const Token &other) const;
 
 	// Returns true if this is the first token on line.
 	// It disregards any leading whitespace.
@@ -88,9 +90,9 @@ struct Token
 
 	// Converts text into numeric value for CONST_INT and CONST_FLOAT token.
 	// Returns false if the parsed value cannot fit into an int or float.
-	bool iValue(int* value) const;
-	bool uValue(unsigned int* value) const;
-	bool fValue(float* value) const;
+	bool iValue(int *value) const;
+	bool uValue(unsigned int *value) const;
+	bool fValue(float *value) const;
 
 	int type;
 	unsigned int flags;
@@ -98,17 +100,17 @@ struct Token
 	std::string text;
 };
 
-inline bool operator==(const Token& lhs, const Token& rhs)
+inline bool operator==(const Token &lhs, const Token &rhs)
 {
 	return lhs.equals(rhs);
 }
 
-inline bool operator!=(const Token& lhs, const Token& rhs)
+inline bool operator!=(const Token &lhs, const Token &rhs)
 {
 	return !lhs.equals(rhs);
 }
 
-extern std::ostream& operator<<(std::ostream& out, const Token& token);
+std::ostream &operator<<(std::ostream &out, const Token &token);
 
 }  // namepsace pp
 #endif  // COMPILER_PREPROCESSOR_TOKEN_H_

src/OpenGL/compiler/preprocessor/Tokenizer.h

@@ -19,8 +19,6 @@
 #include "Lexer.h"
 #include "pp_utils.h"
 
-#include <algorithm>
-
 namespace pp
 {
 
@@ -31,7 +29,7 @@ class Tokenizer : public Lexer
 public:
 	struct Context
 	{
-		Diagnostics* diagnostics;
+		Diagnostics *diagnostics;
 
 		Input input;
 		// The location where yytext points to. Token location should track
@@ -42,25 +40,26 @@ public:
 		bool leadingSpace;
 		bool lineStart;
 	};
-	static const size_t kMaxTokenLength;
 
-	Tokenizer(Diagnostics* diagnostics);
-	~Tokenizer();
+	Tokenizer(Diagnostics *diagnostics);
+	~Tokenizer() override;
 
-	bool init(int count, const char* const string[], const int length[]);
+	bool init(size_t count, const char *const string[], const int length[]);
 
 	void setFileNumber(int file);
 	void setLineNumber(int line);
+	void setMaxTokenSize(size_t maxTokenSize);
 
-	virtual void lex(Token* token);
+	void lex(Token *token) override;
 
 private:
 	PP_DISALLOW_COPY_AND_ASSIGN(Tokenizer);
 	bool initScanner();
 	void destroyScanner();
 
-	void* mHandle;  // Scanner handle.
+	void *mHandle;  // Scanner handle.
 	Context mContext;  // Scanner extra.
+	size_t mMaxTokenSize;  // Maximum token size
 };
 
 }  // namespace pp

src/OpenGL/compiler/preprocessor/Tokenizer.l

@@ -39,14 +39,27 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
 }
 
 %{
+#if defined(_MSC_VER)
+#pragma warning(disable: 4005)
+#endif
+
 #include "Tokenizer.h"
 
-#include "Diagnostics.h"
+#include "DiagnosticsBase.h"
 #include "Token.h"
 
 #if defined(__GNUC__)
 // Triggered by the auto-generated yy_fatal_error function.
 #pragma GCC diagnostic ignored "-Wmissing-noreturn"
+#elif defined(_MSC_VER)
+#pragma warning(disable: 4244)
+#endif
+
+// Workaround for flex using the register keyword, deprecated in C++11.
+#ifdef __cplusplus
+#if __cplusplus > 199711L
+#define register
+#endif
 #endif
 
 typedef std::string YYSTYPE;
@@ -80,7 +93,7 @@ typedef pp::SourceLocation YYLTYPE;
     } while(0);
 
 #define YY_INPUT(buf, result, maxSize) \
-    result = yyextra->input.read(buf, maxSize);
+    result = yyextra->input.read(buf, maxSize, &yylineno);
 
 %}
 
@@ -109,11 +122,18 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 
     /* Block comment */
     /* Line breaks are just counted - not returned. */
-    /* The comment is replaced by a single space. */ 
+    /* The comment is replaced by a single space. */
 "/*" { BEGIN(COMMENT); }
 <COMMENT>[^*\r\n]+
 <COMMENT>"*"
-<COMMENT>{NEWLINE} { ++yylineno; }
+<COMMENT>{NEWLINE} {
+    if (yylineno == INT_MAX)
+    {
+        *yylval = "Integer overflow on line number";
+        return pp::Token::GOT_ERROR;
+    }
+    ++yylineno;
+}
 <COMMENT>"*/" {
     yyextra->leadingSpace = true;
     BEGIN(INITIAL);
@@ -240,13 +260,16 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 [ \t\v\f]+   { yyextra->leadingSpace = true; }
 
 {NEWLINE} {
+    if (yylineno == INT_MAX)
+    {
+        *yylval = "Integer overflow on line number";
+        return pp::Token::GOT_ERROR;
+    }
     ++yylineno;
     yylval->assign(1, '\n');
     return '\n';
 }
 
-\\{NEWLINE} { ++yylineno; }
-
 . {
     yylval->assign(1, yytext[0]);
     return pp::Token::PP_OTHER;
@@ -257,23 +280,30 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
     // Set the location for EOF token manually.
     pp::Input* input = &yyextra->input;
     pp::Input::Location* scanLoc = &yyextra->scanLoc;
-    int sIndexMax = std::max(0, input->count() - 1);
+    yy_size_t sIndexMax = input->count() ? input->count() - 1 : 0;
     if (scanLoc->sIndex != sIndexMax)
     {
         // We can only reach here if there are empty strings at the
         // end of the input.
         scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0;
-        yyfileno = sIndexMax; yylineno = 1;
+        // FIXME: this is not 64-bit clean.
+        yyfileno = static_cast<int>(sIndexMax); yylineno = 1;
     }
     yylloc->file = yyfileno;
     yylloc->line = yylineno;
     yylval->clear();
 
-    if (YY_START == COMMENT)
+    // Line number overflows fake EOFs to exit early, check for this case.
+    if (yylineno == INT_MAX) {
+        yyextra->diagnostics->report(pp::Diagnostics::PP_TOKENIZER_ERROR,
+                pp::SourceLocation(yyfileno, yylineno),
+                "Integer overflow on line number");
+    }
+    else if (YY_START == COMMENT)
     {
-        yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
+        yyextra->diagnostics->report(pp::Diagnostics::PP_EOF_IN_COMMENT,
                                      pp::SourceLocation(yyfileno, yylineno),
-                                     "");
+                                     "EOF while in a comment");
     }
     yyterminate();
 }
@@ -282,11 +312,7 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 
 namespace pp {
 
-// TODO(alokp): Maximum token length should ideally be specified by
-// the preprocessor client, i.e., the compiler.
-const size_t Tokenizer::kMaxTokenLength = 1024;
-
-Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
+Tokenizer::Tokenizer(Diagnostics *diagnostics) : mHandle(nullptr), mMaxTokenSize(1024)
 {
     mContext.diagnostics = diagnostics;
 }
@@ -296,10 +322,10 @@ Tokenizer::~Tokenizer()
     destroyScanner();
 }
 
-bool Tokenizer::init(int count, const char* const string[], const int length[])
+bool Tokenizer::init(size_t count, const char * const string[], const int length[])
 {
-    if (count < 0) return false;
-    if ((count > 0) && (string == 0)) return false;
+    if ((count > 0) && (string == 0))
+        return false;
 
     mContext.input = Input(count, string, length);
     return initScanner();
@@ -317,14 +343,30 @@ void Tokenizer::setLineNumber(int line)
     yyset_lineno(line, mHandle);
 }
 
-void Tokenizer::lex(Token* token)
+void Tokenizer::setMaxTokenSize(size_t maxTokenSize)
+{
+    mMaxTokenSize = maxTokenSize;
+}
+
+void Tokenizer::lex(Token *token)
 {
-    token->type = yylex(&token->text, &token->location, mHandle);
-    if (token->text.size() > kMaxTokenLength)
+    int tokenType = yylex(&token->text, &token->location, mHandle);
+
+    if (tokenType == Token::GOT_ERROR)
+    {
+        mContext.diagnostics->report(Diagnostics::PP_TOKENIZER_ERROR, token->location, token->text);
+        token->type = Token::LAST;
+    }
+    else
+    {
+        token->type = tokenType;
+    }
+
+    if (token->text.size() > mMaxTokenSize)
     {
-        mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG,
+        mContext.diagnostics->report(Diagnostics::PP_TOKEN_TOO_LONG,
                                      token->location, token->text);
-        token->text.erase(kMaxTokenLength);
+        token->text.erase(mMaxTokenSize);
     }
 
     token->flags = 0;
@@ -338,7 +380,7 @@ void Tokenizer::lex(Token* token)
 
 bool Tokenizer::initScanner()
 {
-    if ((mHandle == NULL) && yylex_init_extra(&mContext, &mHandle))
+    if ((mHandle == nullptr) && yylex_init_extra(&mContext, &mHandle))
         return false;
 
     yyrestart(0, mHandle);
@@ -347,11 +389,11 @@ bool Tokenizer::initScanner()
 
 void Tokenizer::destroyScanner()
 {
-    if (mHandle == NULL)
+    if (mHandle == nullptr)
         return;
 
     yylex_destroy(mHandle);
-    mHandle = NULL;
+    mHandle = nullptr;
 }
 
 }  // namespace pp

src/OpenGL/compiler/preprocessor/numeric_lex.h

@@ -62,7 +62,7 @@ bool numeric_lex_float(const std::string& str, FloatType* value)
 	stream.imbue(std::locale::classic());
 
 	stream >> (*value);
-	return !stream.fail();
+	return !stream.fail() && std::isfinite(*value);
 }
 
 } // namespace pp.

src/OpenGL/compiler/preprocessor/preprocessor.vcxproj

@@ -143,8 +143,8 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
-    <ClInclude Include="Diagnostics.h" />
-    <ClInclude Include="DirectiveHandler.h" />
+    <ClInclude Include="DiagnosticsBase.h" />
+    <ClInclude Include="DirectiveHandlerBase.h" />
     <ClInclude Include="DirectiveParser.h" />
     <ClInclude Include="ExpressionParser.h" />
     <ClInclude Include="Input.h" />
@@ -160,8 +160,8 @@
     <ClInclude Include="Tokenizer.h" />
   </ItemGroup>
   <ItemGroup>
-    <ClCompile Include="Diagnostics.cpp" />
-    <ClCompile Include="DirectiveHandler.cpp" />
+    <ClCompile Include="DiagnosticsBase.cpp" />
+    <ClCompile Include="DirectiveHandlerBase.cpp" />
     <ClCompile Include="DirectiveParser.cpp" />
     <ClCompile Include="ExpressionParser.cpp" />
     <ClCompile Include="Input.cpp" />

src/OpenGL/compiler/preprocessor/preprocessor.vcxproj.filters

@@ -15,12 +15,6 @@
     </Filter>
   </ItemGroup>
   <ItemGroup>
-    <ClInclude Include="Diagnostics.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="DirectiveHandler.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
     <ClInclude Include="DirectiveParser.h">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -60,14 +54,14 @@
     <ClInclude Include="Tokenizer.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="DiagnosticsBase.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+    <ClInclude Include="DirectiveHandlerBase.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
-    <ClCompile Include="Diagnostics.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="DirectiveHandler.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="DirectiveParser.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -95,9 +89,15 @@
     <ClCompile Include="Tokenizer.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="DiagnosticsBase.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="DirectiveHandlerBase.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <None Include="Tokenizer.l" />
     <None Include="ExpressionParser.y" />
   </ItemGroup>
-</Project>
+</Project>
\ No newline at end of file