Skip to content

G
glslang
Commit 6ab3d582 by John Kessenich
Options

Preprocessor: 1) Rationalize the "fixed atom" scheme, 2) remove redundant…

Preprocessor: 1) Rationalize the "fixed atom" scheme, 2) remove redundant lookups when the text is already available. This simplification is a prelude to eliminating what I appear unnecessary symbol inserts into tables when tokenizing in the preprecessor, which show up as taking notable time. (Performance issue.) It also simply makes the preprocessor easier to understand, which it is badly in need of.
parent 2f273369
Showing with 237 additions and 269 deletions
glslang/Include/Common.h
...@@ -92,6 +92,7 @@ namespace glslang { ...@@ -92,6 +92,7 @@ namespace glslang {
typedef pool_allocator<char> TStringAllocator; typedef pool_allocator<char> TStringAllocator;
typedef std::basic_string <char, std::char_traits<char>, TStringAllocator> TString; typedef std::basic_string <char, std::char_traits<char>, TStringAllocator> TString;
// Repackage the std::hash for use by unordered map/set with a TString key.
struct TStringHash { struct TStringHash {
size_t operator()(const TString& string) const { return std::hash<TString>()(string); } size_t operator()(const TString& string) const { return std::hash<TString>()(string); }
}; };
......
glslang/MachineIndependent/ParseHelper.h
...@@ -306,7 +306,7 @@ protected: ...@@ -306,7 +306,7 @@ protected:
TInputScanner* currentScanner; TInputScanner* currentScanner;
int numErrors; // number of compile-time errors encountered int numErrors; // number of compile-time errors encountered
bool parsingBuiltins; // true if parsing built-in symbols/functions bool parsingBuiltins; // true if parsing built-in symbols/functions
std::unordered_map<TString, TExtensionBehavior> extensionBehavior; // for each extension string, what its current behavior is set to TUnorderedMap<TString, TExtensionBehavior> extensionBehavior; // for each extension string, what its current behavior is set to
static const int maxSamplerIndex = EsdNumDims * (EbtNumTypes * (2 * 2 * 2)); // see computeSamplerTypeIndex() static const int maxSamplerIndex = EsdNumDims * (EbtNumTypes * (2 * 2 * 2)); // see computeSamplerTypeIndex()
TPrecisionQualifier defaultSamplerPrecision[maxSamplerIndex]; TPrecisionQualifier defaultSamplerPrecision[maxSamplerIndex];
bool afterEOF; bool afterEOF;
......
glslang/MachineIndependent/Scan.cpp
...@@ -567,33 +567,38 @@ int TScanContext::tokenize(TPpContext* pp, TParserToken& token) ...@@ -567,33 +567,38 @@ int TScanContext::tokenize(TPpContext* pp, TParserToken& token)
parseContext.error(loc, "illegal use of escape character", "\\", ""); parseContext.error(loc, "illegal use of escape character", "\\", "");
break; break;
case CPP_AND_OP: return AND_OP; case PpAtomAdd: return ADD_ASSIGN;
case CPP_SUB_ASSIGN: return SUB_ASSIGN; case PpAtomSub: return SUB_ASSIGN;
case CPP_MOD_ASSIGN: return MOD_ASSIGN; case PpAtomMul: return MUL_ASSIGN;
case CPP_ADD_ASSIGN: return ADD_ASSIGN; case PpAtomDiv: return DIV_ASSIGN;
case CPP_DIV_ASSIGN: return DIV_ASSIGN; case PpAtomMod: return MOD_ASSIGN;
case CPP_MUL_ASSIGN: return MUL_ASSIGN;
case CPP_EQ_OP: return EQ_OP; case PpAtomRight: return RIGHT_OP;
case CPP_XOR_OP: return XOR_OP; case PpAtomLeft: return LEFT_OP;
case CPP_GE_OP: return GE_OP;
case CPP_RIGHT_OP: return RIGHT_OP; case PpAtomRightAssign: return RIGHT_ASSIGN;
case CPP_LE_OP: return LE_OP; case PpAtomLeftAssign: return LEFT_ASSIGN;
case CPP_LEFT_OP: return LEFT_OP; case PpAtomAndAssign: return AND_ASSIGN;
case CPP_DEC_OP: return DEC_OP; case PpAtomOrAssign: return OR_ASSIGN;
case CPP_NE_OP: return NE_OP; case PpAtomXorAssign: return XOR_ASSIGN;
case CPP_OR_OP: return OR_OP;
case CPP_INC_OP: return INC_OP; case PpAtomAnd: return AND_OP;
case CPP_RIGHT_ASSIGN: return RIGHT_ASSIGN; case PpAtomOr: return OR_OP;
case CPP_LEFT_ASSIGN: return LEFT_ASSIGN; case PpAtomXor: return XOR_OP;
case CPP_AND_ASSIGN: return AND_ASSIGN;
case CPP_OR_ASSIGN: return OR_ASSIGN; case PpAtomEQ: return EQ_OP;
case CPP_XOR_ASSIGN: return XOR_ASSIGN; case PpAtomGE: return GE_OP;
case PpAtomNE: return NE_OP;
case PpAtomLE: return LE_OP;
case PpAtomDecrement: return DEC_OP;
case PpAtomIncrement: return INC_OP;
case CPP_INTCONSTANT: parserToken->sType.lex.i = ppToken.ival; return INTCONSTANT; case PpAtomConstInt: parserToken->sType.lex.i = ppToken.ival; return INTCONSTANT;
case CPP_UINTCONSTANT: parserToken->sType.lex.i = ppToken.ival; return UINTCONSTANT; case PpAtomConstUint: parserToken->sType.lex.i = ppToken.ival; return UINTCONSTANT;
case CPP_FLOATCONSTANT: parserToken->sType.lex.d = ppToken.dval; return FLOATCONSTANT; case PpAtomConstFloat: parserToken->sType.lex.d = ppToken.dval; return FLOATCONSTANT;
case CPP_DOUBLECONSTANT: parserToken->sType.lex.d = ppToken.dval; return DOUBLECONSTANT; case PpAtomConstDouble: parserToken->sType.lex.d = ppToken.dval; return DOUBLECONSTANT;
case CPP_IDENTIFIER: return tokenizeIdentifier(); case PpAtomIdentifier: return tokenizeIdentifier();
case EOF: return 0; case EOF: return 0;
......
glslang/MachineIndependent/preprocessor/PpAtom.cpp
...@@ -98,27 +98,28 @@ const struct { ...@@ -98,27 +98,28 @@ const struct {
int val; int val;
const char* str; const char* str;
} tokens[] = { } tokens[] = {
{ CPP_AND_OP, "&&" }, { PpAtomDefine, "define" },
{ CPP_AND_ASSIGN, "&=" }, { PpAtomDefined, "defined" },
{ CPP_SUB_ASSIGN, "-=" }, { PpAtomUndef, "undef" },
{ CPP_MOD_ASSIGN, "%=" }, { PpAtomIf, "if" },
{ CPP_ADD_ASSIGN, "+=" }, { PpAtomElif, "elif" },
{ CPP_DIV_ASSIGN, "/=" }, { PpAtomElse, "else" },
{ CPP_MUL_ASSIGN, "*=" }, { PpAtomEndif, "endif" },
{ CPP_EQ_OP, "==" }, { PpAtomIfdef, "ifdef" },
{ CPP_XOR_OP, "^^" }, { PpAtomIfndef, "ifndef" },
{ CPP_XOR_ASSIGN, "^=" }, { PpAtomLine, "line" },
{ CPP_GE_OP, ">=" }, { PpAtomPragma, "pragma" },
{ CPP_RIGHT_OP, ">>" }, { PpAtomError, "error" },
{ CPP_RIGHT_ASSIGN, ">>="},
{ CPP_LE_OP, "<=" }, { PpAtomVersion, "version" },
{ CPP_LEFT_OP, "<<" }, { PpAtomCore, "core" },
{ CPP_LEFT_ASSIGN, "<<="}, { PpAtomCompatibility, "compatibility" },
{ CPP_DEC_OP, "--" }, { PpAtomEs, "es" },
{ CPP_NE_OP, "!=" }, { PpAtomExtension, "extension" },
{ CPP_OR_OP, "||" },
{ CPP_OR_ASSIGN, "|=" }, { PpAtomLineMacro, "__LINE__" },
{ CPP_INC_OP, "++" }, { PpAtomFileMacro, "__FILE__" },
{ PpAtomVersionMacro, "__VERSION__" },
}; };
} // end anonymous namespace } // end anonymous namespace
...@@ -131,9 +132,10 @@ namespace glslang { ...@@ -131,9 +132,10 @@ namespace glslang {
int TPpContext::LookUpAddString(const char* s) int TPpContext::LookUpAddString(const char* s)
{ {
auto it = atomMap.find(s); auto it = atomMap.find(s);
if (it == atomMap.end()) if (it == atomMap.end()) {
return AddAtomFixed(s, nextAtom++); AddAtomFixed(s, nextAtom);
else return nextAtom++;
} else
return it->second; return it->second;
} }
...@@ -159,14 +161,12 @@ const char* TPpContext::GetAtomString(int atom) ...@@ -159,14 +161,12 @@ const char* TPpContext::GetAtomString(int atom)
// //
// Add forced mapping of string to atom. // Add forced mapping of string to atom.
// //
int TPpContext::AddAtomFixed(const char* s, int atom) void TPpContext::AddAtomFixed(const char* s, int atom)
{ {
auto it = atomMap.insert(std::pair<TString, int>(s, atom)).first; auto it = atomMap.insert(std::pair<TString, int>(s, atom)).first;
if (stringMap.size() < (size_t)atom + 1) if (stringMap.size() < (size_t)atom + 1)
stringMap.resize(atom + 100, 0); stringMap.resize(atom + 100, 0);
stringMap[atom] = &it->first; stringMap[atom] = &it->first;
return atom;
} }
// //
...@@ -189,7 +189,7 @@ void TPpContext::InitAtomTable() ...@@ -189,7 +189,7 @@ void TPpContext::InitAtomTable()
for (int ii = 0; ii < sizeof(tokens)/sizeof(tokens[0]); ii++) for (int ii = 0; ii < sizeof(tokens)/sizeof(tokens[0]); ii++)
AddAtomFixed(tokens[ii].str, tokens[ii].val); AddAtomFixed(tokens[ii].str, tokens[ii].val);
nextAtom = CPP_FIRST_USER_TOKEN_SY; nextAtom = PpAtomLast;
} }
} // end namespace glslang } // end namespace glslang
glslang/MachineIndependent/preprocessor/PpContext.cpp
...@@ -93,48 +93,6 @@ TPpContext::TPpContext(TParseContext& pc) : ...@@ -93,48 +93,6 @@ TPpContext::TPpContext(TParseContext& pc) :
for (elsetracker = 0; elsetracker < maxIfNesting; elsetracker++) for (elsetracker = 0; elsetracker < maxIfNesting; elsetracker++)
elseSeen[elsetracker] = false; elseSeen[elsetracker] = false;
elsetracker = 0; elsetracker = 0;
// The following identifies all legal characters in GLSL:
//for (int c = 0; c < 256; ++c)
// languageCharacters[c] = false;
//for (int c = 'a'; c <= 'z'; ++c)
// languageCharacters[c] = true;
//for (int c = 'A'; c <= 'Z'; ++c)
// languageCharacters[c] = true;
//languageCharacters['_'] = true;
//for (int c = '0'; c <= '9'; ++c)
// languageCharacters[c] = true;
//languageCharacters['.'] = true;
//languageCharacters['+'] = true;
//languageCharacters['-'] = true;
//languageCharacters['/'] = true;
//languageCharacters['*'] = true;
//languageCharacters['%'] = true;
//languageCharacters['<'] = true;
//languageCharacters['>'] = true;
//languageCharacters['['] = true;
//languageCharacters[']'] = true;
//languageCharacters['('] = true;
//languageCharacters[')'] = true;
//languageCharacters['{'] = true;
//languageCharacters['}'] = true;
//languageCharacters['^'] = true;
//languageCharacters['|'] = true;
//languageCharacters['&'] = true;
//languageCharacters['~'] = true;
//languageCharacters['='] = true;
//languageCharacters['!'] = true;
//languageCharacters[':'] = true;
//languageCharacters[';'] = true;
//languageCharacters[','] = true;
//languageCharacters['?'] = true;
//languageCharacters['#'] = true;
//// white space
//languageCharacters[' '] = true;
//for (int c = 9; c <= 13; ++c)
// languageCharacters[c] = true;
} }
TPpContext::~TPpContext() TPpContext::~TPpContext()
......
glslang/MachineIndependent/preprocessor/PpContext.h
...@@ -287,31 +287,6 @@ protected: ...@@ -287,31 +287,6 @@ protected:
// //
// from Pp.cpp // from Pp.cpp
// //
int bindAtom;
int constAtom;
int defaultAtom;
int defineAtom;
int definedAtom;
int elseAtom;
int elifAtom;
int endifAtom;
int ifAtom;
int ifdefAtom;
int ifndefAtom;
int includeAtom;
int lineAtom;
int pragmaAtom;
int texunitAtom;
int undefAtom;
int errorAtom;
int __LINE__Atom;
int __FILE__Atom;
int __VERSION__Atom;
int versionAtom;
int coreAtom;
int compatibilityAtom;
int esAtom;
int extensionAtom;
TSourceLoc ifloc; /* outermost #if */ TSourceLoc ifloc; /* outermost #if */
int InitCPP(); int InitCPP();
...@@ -467,7 +442,7 @@ protected: ...@@ -467,7 +442,7 @@ protected:
TStringMap stringMap; TStringMap stringMap;
int nextAtom; int nextAtom;
void InitAtomTable(); void InitAtomTable();
int AddAtomFixed(const char* s, int atom); void AddAtomFixed(const char* s, int atom);
int LookUpAddString(const char* s); int LookUpAddString(const char* s);
const char* GetAtomString(int atom); const char* GetAtomString(int atom);
......
glslang/MachineIndependent/preprocessor/PpTokens.cpp
...@@ -126,25 +126,25 @@ void TPpContext::RecordToken(TokenStream *pTok, int token, TPpToken* ppToken) ...@@ -126,25 +126,25 @@ void TPpContext::RecordToken(TokenStream *pTok, int token, TPpToken* ppToken)
const char* s; const char* s;
char* str = NULL; char* str = NULL;
if (token > 256) if (token > PpAtomMaxSingle)
lAddByte(pTok, (unsigned char)((token & 0x7f) + 0x80)); lAddByte(pTok, (unsigned char)((token & 0x7f) + 0x80));
else else
lAddByte(pTok, (unsigned char)(token & 0x7f)); lAddByte(pTok, (unsigned char)(token & 0x7f));
switch (token) { switch (token) {
case CPP_IDENTIFIER: case PpAtomIdentifier:
case CPP_STRCONSTANT: case PpAtomConstString:
s = GetAtomString(ppToken->atom); s = ppToken->name;
while (*s) while (*s)
lAddByte(pTok, (unsigned char) *s++); lAddByte(pTok, (unsigned char) *s++);
lAddByte(pTok, 0); lAddByte(pTok, 0);
break; break;
case CPP_INTCONSTANT: case PpAtomConstInt:
case CPP_UINTCONSTANT: case PpAtomConstUint:
case CPP_FLOATCONSTANT: case PpAtomConstFloat:
case CPP_DOUBLECONSTANT: case PpAtomConstDouble:
str = ppToken->name; str = ppToken->name;
while (*str){ while (*str) {
lAddByte(pTok, (unsigned char) *str); lAddByte(pTok, (unsigned char) *str);
str++; str++;
} }
...@@ -168,7 +168,7 @@ void TPpContext::RewindTokenStream(TokenStream *pTok) ...@@ -168,7 +168,7 @@ void TPpContext::RewindTokenStream(TokenStream *pTok)
*/ */
int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken) int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken)
{ {
char tokenText[TPpToken::maxTokenLength + 1]; char* tokenText = ppToken->name;
int ltoken, len; int ltoken, len;
int ch; int ch;
...@@ -182,17 +182,17 @@ int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken) ...@@ -182,17 +182,17 @@ int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken)
parseContext.requireProfile(ppToken->loc, ~EEsProfile, "token pasting (##)"); parseContext.requireProfile(ppToken->loc, ~EEsProfile, "token pasting (##)");
parseContext.profileRequires(ppToken->loc, ~EEsProfile, 130, 0, "token pasting (##)"); parseContext.profileRequires(ppToken->loc, ~EEsProfile, 130, 0, "token pasting (##)");
parseContext.error(ppToken->loc, "token pasting not implemented (internal error)", "##", ""); parseContext.error(ppToken->loc, "token pasting not implemented (internal error)", "##", "");
//return CPP_TOKEN_PASTE; //return PpAtomPaste;
return ReadToken(pTok, ppToken); return ReadToken(pTok, ppToken);
} else } else
lUnreadByte(pTok); lUnreadByte(pTok);
break; break;
case CPP_STRCONSTANT: case PpAtomConstString:
case CPP_IDENTIFIER: case PpAtomIdentifier:
case CPP_FLOATCONSTANT: case PpAtomConstFloat:
case CPP_DOUBLECONSTANT: case PpAtomConstDouble:
case CPP_INTCONSTANT: case PpAtomConstInt:
case CPP_UINTCONSTANT: case PpAtomConstUint:
len = 0; len = 0;
ch = lReadByte(pTok); ch = lReadByte(pTok);
while (ch != 0) { while (ch != 0) {
...@@ -208,17 +208,18 @@ int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken) ...@@ -208,17 +208,18 @@ int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken)
tokenText[len] = 0; tokenText[len] = 0;
switch (ltoken) { switch (ltoken) {
case CPP_IDENTIFIER: case PpAtomIdentifier:
case CPP_STRCONSTANT:
ppToken->atom = LookUpAddString(tokenText); ppToken->atom = LookUpAddString(tokenText);
break; break;
case CPP_FLOATCONSTANT: case PpAtomConstString:
case CPP_DOUBLECONSTANT: break;
case PpAtomConstFloat:
case PpAtomConstDouble:
strcpy(ppToken->name, tokenText); strcpy(ppToken->name, tokenText);
ppToken->dval = atof(ppToken->name); ppToken->dval = atof(ppToken->name);
break; break;
case CPP_INTCONSTANT: case PpAtomConstInt:
case CPP_UINTCONSTANT: case PpAtomConstUint:
strcpy(ppToken->name, tokenText); strcpy(ppToken->name, tokenText);
if (len > 0 && tokenText[0] == '0') { if (len > 0 && tokenText[0] == '0') {
if (len > 1 && (tokenText[1] == 'x' || tokenText[1] == 'X')) if (len > 1 && (tokenText[1] == 'x' || tokenText[1] == 'X'))
......
glslang/MachineIndependent/preprocessor/PpTokens.h
...@@ -78,40 +78,89 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ...@@ -78,40 +78,89 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef PARSER_H #ifndef PARSER_H
#define PARSER_H #define PARSER_H
#define CPP_AND_OP 257 namespace glslang {
#define CPP_SUB_ASSIGN 259
#define CPP_MOD_ASSIGN 260 // Multi-character tokens
#define CPP_ADD_ASSIGN 261 enum EFixedAtoms {
#define CPP_DIV_ASSIGN 262 PpAtomMaxSingle = 256, // single character tokens get their own char value as their token, skip them
#define CPP_MUL_ASSIGN 263
#define CPP_EQ_OP 264 // Operators
#define CPP_XOR_OP 265
#define CPP_ERROR_SY 266 PpAtomAdd,
#define CPP_FLOATCONSTANT 267 PpAtomSub,
#define CPP_GE_OP 268 PpAtomMul,
#define CPP_RIGHT_OP 269 PpAtomDiv,
#define CPP_IDENTIFIER 270 PpAtomMod,
#define CPP_INTCONSTANT 271
#define CPP_LE_OP 272 PpAtomRight,
#define CPP_LEFT_OP 273 PpAtomLeft,
#define CPP_DEC_OP 274
#define CPP_NE_OP 275 PpAtomRightAssign,
#define CPP_OR_OP 276 PpAtomLeftAssign,
#define CPP_INC_OP 277 PpAtomAndAssign,
#define CPP_STRCONSTANT 278 PpAtomOrAssign,
#define CPP_RIGHT_ASSIGN 280 PpAtomXorAssign,
#define CPP_LEFT_ASSIGN 281
#define CPP_AND_ASSIGN 282 PpAtomAnd,
#define CPP_OR_ASSIGN 283 PpAtomOr,
#define CPP_XOR_ASSIGN 284 PpAtomXor,
#define CPP_LEFT_BRACKET 285
#define CPP_RIGHT_BRACKET 286 PpAtomEQ,
#define CPP_LEFT_BRACE 287 PpAtomNE,
#define CPP_RIGHT_BRACE 288 PpAtomGE,
#define CPP_UINTCONSTANT 289 PpAtomLE,
#define CPP_DOUBLECONSTANT 290
#define CPP_TOKEN_PASTE 291 PpAtomDecrement,
PpAtomIncrement,
#define CPP_FIRST_USER_TOKEN_SY 292
PpAtomPaste,
// Constants
PpAtomConstInt,
PpAtomConstUint,
PpAtomConstFloat,
PpAtomConstDouble,
PpAtomConstString,
// Indentifiers
PpAtomIdentifier,
// preprocessor "keywords"
PpAtomDefine,
PpAtomDefined,
PpAtomUndef,
PpAtomIf,
PpAtomIfdef,
PpAtomIfndef,
PpAtomElse,
PpAtomElif,
PpAtomEndif,
PpAtomLine,
PpAtomPragma,
PpAtomError,
// #version ...
PpAtomVersion,
PpAtomCore,
PpAtomCompatibility,
PpAtomEs,
// #extension
PpAtomExtension,
// __LINE__, __FILE__, __VERSION__
PpAtomLineMacro,
PpAtomFileMacro,
PpAtomVersionMacro,
PpAtomLast,
};
} // end namespace glslang
#endif /* not PARSER_H */ #endif /* not PARSER_H */
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment