Skip to content

G
glslang
Commit 6ab3d582 by John Kessenich
Options

Preprocessor: 1) Rationalize the "fixed atom" scheme, 2) remove redundant…

Preprocessor: 1) Rationalize the "fixed atom" scheme, 2) remove redundant lookups when the text is already available. This simplification is a prelude to eliminating what I appear unnecessary symbol inserts into tables when tokenizing in the preprecessor, which show up as taking notable time. (Performance issue.) It also simply makes the preprocessor easier to understand, which it is badly in need of.
parent 2f273369
Showing with 237 additions and 269 deletions
glslang/Include/Common.h
......@@ -92,6 +92,7 @@ namespace glslang {
typedef pool_allocator<char> TStringAllocator;
typedef std::basic_string <char, std::char_traits<char>, TStringAllocator> TString;
// Repackage the std::hash for use by unordered map/set with a TString key.
struct TStringHash {
size_t operator()(const TString& string) const { return std::hash<TString>()(string); }
};
......
glslang/MachineIndependent/ParseHelper.h
......@@ -306,7 +306,7 @@ protected:
TInputScanner* currentScanner;
int numErrors; // number of compile-time errors encountered
bool parsingBuiltins; // true if parsing built-in symbols/functions
std::unordered_map<TString, TExtensionBehavior> extensionBehavior; // for each extension string, what its current behavior is set to
TUnorderedMap<TString, TExtensionBehavior> extensionBehavior; // for each extension string, what its current behavior is set to
static const int maxSamplerIndex = EsdNumDims * (EbtNumTypes * (2 * 2 * 2)); // see computeSamplerTypeIndex()
TPrecisionQualifier defaultSamplerPrecision[maxSamplerIndex];
bool afterEOF;
......
glslang/MachineIndependent/Scan.cpp
......@@ -567,33 +567,38 @@ int TScanContext::tokenize(TPpContext* pp, TParserToken& token)
parseContext.error(loc, "illegal use of escape character", "\\", "");
break;
case CPP_AND_OP: return AND_OP;
case CPP_SUB_ASSIGN: return SUB_ASSIGN;
case CPP_MOD_ASSIGN: return MOD_ASSIGN;
case CPP_ADD_ASSIGN: return ADD_ASSIGN;
case CPP_DIV_ASSIGN: return DIV_ASSIGN;
case CPP_MUL_ASSIGN: return MUL_ASSIGN;
case CPP_EQ_OP: return EQ_OP;
case CPP_XOR_OP: return XOR_OP;
case CPP_GE_OP: return GE_OP;
case CPP_RIGHT_OP: return RIGHT_OP;
case CPP_LE_OP: return LE_OP;
case CPP_LEFT_OP: return LEFT_OP;
case CPP_DEC_OP: return DEC_OP;
case CPP_NE_OP: return NE_OP;
case CPP_OR_OP: return OR_OP;
case CPP_INC_OP: return INC_OP;
case CPP_RIGHT_ASSIGN: return RIGHT_ASSIGN;
case CPP_LEFT_ASSIGN: return LEFT_ASSIGN;
case CPP_AND_ASSIGN: return AND_ASSIGN;
case CPP_OR_ASSIGN: return OR_ASSIGN;
case CPP_XOR_ASSIGN: return XOR_ASSIGN;
case PpAtomAdd: return ADD_ASSIGN;
case PpAtomSub: return SUB_ASSIGN;
case PpAtomMul: return MUL_ASSIGN;
case PpAtomDiv: return DIV_ASSIGN;
case PpAtomMod: return MOD_ASSIGN;
case PpAtomRight: return RIGHT_OP;
case PpAtomLeft: return LEFT_OP;
case PpAtomRightAssign: return RIGHT_ASSIGN;
case PpAtomLeftAssign: return LEFT_ASSIGN;
case PpAtomAndAssign: return AND_ASSIGN;
case PpAtomOrAssign: return OR_ASSIGN;
case PpAtomXorAssign: return XOR_ASSIGN;
case PpAtomAnd: return AND_OP;
case PpAtomOr: return OR_OP;
case PpAtomXor: return XOR_OP;
case PpAtomEQ: return EQ_OP;
case PpAtomGE: return GE_OP;
case PpAtomNE: return NE_OP;
case PpAtomLE: return LE_OP;
case PpAtomDecrement: return DEC_OP;
case PpAtomIncrement: return INC_OP;
case CPP_INTCONSTANT: parserToken->sType.lex.i = ppToken.ival; return INTCONSTANT;
case CPP_UINTCONSTANT: parserToken->sType.lex.i = ppToken.ival; return UINTCONSTANT;
case CPP_FLOATCONSTANT: parserToken->sType.lex.d = ppToken.dval; return FLOATCONSTANT;
case CPP_DOUBLECONSTANT: parserToken->sType.lex.d = ppToken.dval; return DOUBLECONSTANT;
case CPP_IDENTIFIER: return tokenizeIdentifier();
case PpAtomConstInt: parserToken->sType.lex.i = ppToken.ival; return INTCONSTANT;
case PpAtomConstUint: parserToken->sType.lex.i = ppToken.ival; return UINTCONSTANT;
case PpAtomConstFloat: parserToken->sType.lex.d = ppToken.dval; return FLOATCONSTANT;
case PpAtomConstDouble: parserToken->sType.lex.d = ppToken.dval; return DOUBLECONSTANT;
case PpAtomIdentifier: return tokenizeIdentifier();
case EOF: return 0;
......
glslang/MachineIndependent/preprocessor/PpAtom.cpp
......@@ -98,27 +98,28 @@ const struct {
int val;
const char* str;
} tokens[] = {
{ CPP_AND_OP, "&&" },
{ CPP_AND_ASSIGN, "&=" },
{ CPP_SUB_ASSIGN, "-=" },
{ CPP_MOD_ASSIGN, "%=" },
{ CPP_ADD_ASSIGN, "+=" },
{ CPP_DIV_ASSIGN, "/=" },
{ CPP_MUL_ASSIGN, "*=" },
{ CPP_EQ_OP, "==" },
{ CPP_XOR_OP, "^^" },
{ CPP_XOR_ASSIGN, "^=" },
{ CPP_GE_OP, ">=" },
{ CPP_RIGHT_OP, ">>" },
{ CPP_RIGHT_ASSIGN, ">>="},
{ CPP_LE_OP, "<=" },
{ CPP_LEFT_OP, "<<" },
{ CPP_LEFT_ASSIGN, "<<="},
{ CPP_DEC_OP, "--" },
{ CPP_NE_OP, "!=" },
{ CPP_OR_OP, "||" },
{ CPP_OR_ASSIGN, "|=" },
{ CPP_INC_OP, "++" },
{ PpAtomDefine, "define" },
{ PpAtomDefined, "defined" },
{ PpAtomUndef, "undef" },
{ PpAtomIf, "if" },
{ PpAtomElif, "elif" },
{ PpAtomElse, "else" },
{ PpAtomEndif, "endif" },
{ PpAtomIfdef, "ifdef" },
{ PpAtomIfndef, "ifndef" },
{ PpAtomLine, "line" },
{ PpAtomPragma, "pragma" },
{ PpAtomError, "error" },
{ PpAtomVersion, "version" },
{ PpAtomCore, "core" },
{ PpAtomCompatibility, "compatibility" },
{ PpAtomEs, "es" },
{ PpAtomExtension, "extension" },
{ PpAtomLineMacro, "__LINE__" },
{ PpAtomFileMacro, "__FILE__" },
{ PpAtomVersionMacro, "__VERSION__" },
};
} // end anonymous namespace
......@@ -131,9 +132,10 @@ namespace glslang {
int TPpContext::LookUpAddString(const char* s)
{
auto it = atomMap.find(s);
if (it == atomMap.end())
return AddAtomFixed(s, nextAtom++);
else
if (it == atomMap.end()) {
AddAtomFixed(s, nextAtom);
return nextAtom++;
} else
return it->second;
}
......@@ -159,14 +161,12 @@ const char* TPpContext::GetAtomString(int atom)
//
// Add forced mapping of string to atom.
//
int TPpContext::AddAtomFixed(const char* s, int atom)
void TPpContext::AddAtomFixed(const char* s, int atom)
{
auto it = atomMap.insert(std::pair<TString, int>(s, atom)).first;
if (stringMap.size() < (size_t)atom + 1)
stringMap.resize(atom + 100, 0);
stringMap[atom] = &it->first;
return atom;
}
//
......@@ -189,7 +189,7 @@ void TPpContext::InitAtomTable()
for (int ii = 0; ii < sizeof(tokens)/sizeof(tokens[0]); ii++)
AddAtomFixed(tokens[ii].str, tokens[ii].val);
nextAtom = CPP_FIRST_USER_TOKEN_SY;
nextAtom = PpAtomLast;
}
} // end namespace glslang
glslang/MachineIndependent/preprocessor/PpContext.cpp
......@@ -93,48 +93,6 @@ TPpContext::TPpContext(TParseContext& pc) :
for (elsetracker = 0; elsetracker < maxIfNesting; elsetracker++)
elseSeen[elsetracker] = false;
elsetracker = 0;
// The following identifies all legal characters in GLSL:
//for (int c = 0; c < 256; ++c)
// languageCharacters[c] = false;
//for (int c = 'a'; c <= 'z'; ++c)
// languageCharacters[c] = true;
//for (int c = 'A'; c <= 'Z'; ++c)
// languageCharacters[c] = true;
//languageCharacters['_'] = true;
//for (int c = '0'; c <= '9'; ++c)
// languageCharacters[c] = true;
//languageCharacters['.'] = true;
//languageCharacters['+'] = true;
//languageCharacters['-'] = true;
//languageCharacters['/'] = true;
//languageCharacters['*'] = true;
//languageCharacters['%'] = true;
//languageCharacters['<'] = true;
//languageCharacters['>'] = true;
//languageCharacters['['] = true;
//languageCharacters[']'] = true;
//languageCharacters['('] = true;
//languageCharacters[')'] = true;
//languageCharacters['{'] = true;
//languageCharacters['}'] = true;
//languageCharacters['^'] = true;
//languageCharacters['|'] = true;
//languageCharacters['&'] = true;
//languageCharacters['~'] = true;
//languageCharacters['='] = true;
//languageCharacters['!'] = true;
//languageCharacters[':'] = true;
//languageCharacters[';'] = true;
//languageCharacters[','] = true;
//languageCharacters['?'] = true;
//languageCharacters['#'] = true;
//// white space
//languageCharacters[' '] = true;
//for (int c = 9; c <= 13; ++c)
// languageCharacters[c] = true;
}
TPpContext::~TPpContext()
......
glslang/MachineIndependent/preprocessor/PpContext.h
......@@ -287,31 +287,6 @@ protected:
//
// from Pp.cpp
//
int bindAtom;
int constAtom;
int defaultAtom;
int defineAtom;
int definedAtom;
int elseAtom;
int elifAtom;
int endifAtom;
int ifAtom;
int ifdefAtom;
int ifndefAtom;
int includeAtom;
int lineAtom;
int pragmaAtom;
int texunitAtom;
int undefAtom;
int errorAtom;
int __LINE__Atom;
int __FILE__Atom;
int __VERSION__Atom;
int versionAtom;
int coreAtom;
int compatibilityAtom;
int esAtom;
int extensionAtom;
TSourceLoc ifloc; /* outermost #if */
int InitCPP();
......@@ -467,7 +442,7 @@ protected:
TStringMap stringMap;
int nextAtom;
void InitAtomTable();
int AddAtomFixed(const char* s, int atom);
void AddAtomFixed(const char* s, int atom);
int LookUpAddString(const char* s);
const char* GetAtomString(int atom);
......
glslang/MachineIndependent/preprocessor/PpTokens.cpp
......@@ -126,25 +126,25 @@ void TPpContext::RecordToken(TokenStream *pTok, int token, TPpToken* ppToken)
const char* s;
char* str = NULL;
if (token > 256)
if (token > PpAtomMaxSingle)
lAddByte(pTok, (unsigned char)((token & 0x7f) + 0x80));
else
lAddByte(pTok, (unsigned char)(token & 0x7f));
switch (token) {
case CPP_IDENTIFIER:
case CPP_STRCONSTANT:
s = GetAtomString(ppToken->atom);
case PpAtomIdentifier:
case PpAtomConstString:
s = ppToken->name;
while (*s)
lAddByte(pTok, (unsigned char) *s++);
lAddByte(pTok, 0);
break;
case CPP_INTCONSTANT:
case CPP_UINTCONSTANT:
case CPP_FLOATCONSTANT:
case CPP_DOUBLECONSTANT:
case PpAtomConstInt:
case PpAtomConstUint:
case PpAtomConstFloat:
case PpAtomConstDouble:
str = ppToken->name;
while (*str){
while (*str) {
lAddByte(pTok, (unsigned char) *str);
str++;
}
......@@ -168,7 +168,7 @@ void TPpContext::RewindTokenStream(TokenStream *pTok)
*/
int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken)
{
char tokenText[TPpToken::maxTokenLength + 1];
char* tokenText = ppToken->name;
int ltoken, len;
int ch;
......@@ -182,17 +182,17 @@ int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken)
parseContext.requireProfile(ppToken->loc, ~EEsProfile, "token pasting (##)");
parseContext.profileRequires(ppToken->loc, ~EEsProfile, 130, 0, "token pasting (##)");
parseContext.error(ppToken->loc, "token pasting not implemented (internal error)", "##", "");
//return CPP_TOKEN_PASTE;
//return PpAtomPaste;
return ReadToken(pTok, ppToken);
} else
lUnreadByte(pTok);
break;
case CPP_STRCONSTANT:
case CPP_IDENTIFIER:
case CPP_FLOATCONSTANT:
case CPP_DOUBLECONSTANT:
case CPP_INTCONSTANT:
case CPP_UINTCONSTANT:
case PpAtomConstString:
case PpAtomIdentifier:
case PpAtomConstFloat:
case PpAtomConstDouble:
case PpAtomConstInt:
case PpAtomConstUint:
len = 0;
ch = lReadByte(pTok);
while (ch != 0) {
......@@ -208,17 +208,18 @@ int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken)
tokenText[len] = 0;
switch (ltoken) {
case CPP_IDENTIFIER:
case CPP_STRCONSTANT:
case PpAtomIdentifier:
ppToken->atom = LookUpAddString(tokenText);
break;
case CPP_FLOATCONSTANT:
case CPP_DOUBLECONSTANT:
case PpAtomConstString:
break;
case PpAtomConstFloat:
case PpAtomConstDouble:
strcpy(ppToken->name, tokenText);
ppToken->dval = atof(ppToken->name);
break;
case CPP_INTCONSTANT:
case CPP_UINTCONSTANT:
case PpAtomConstInt:
case PpAtomConstUint:
strcpy(ppToken->name, tokenText);
if (len > 0 && tokenText[0] == '0') {
if (len > 1 && (tokenText[1] == 'x' || tokenText[1] == 'X'))
......
glslang/MachineIndependent/preprocessor/PpTokens.h
......@@ -78,40 +78,89 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef PARSER_H
#define PARSER_H
#define CPP_AND_OP 257
#define CPP_SUB_ASSIGN 259
#define CPP_MOD_ASSIGN 260
#define CPP_ADD_ASSIGN 261
#define CPP_DIV_ASSIGN 262
#define CPP_MUL_ASSIGN 263
#define CPP_EQ_OP 264
#define CPP_XOR_OP 265
#define CPP_ERROR_SY 266
#define CPP_FLOATCONSTANT 267
#define CPP_GE_OP 268
#define CPP_RIGHT_OP 269
#define CPP_IDENTIFIER 270
#define CPP_INTCONSTANT 271
#define CPP_LE_OP 272
#define CPP_LEFT_OP 273
#define CPP_DEC_OP 274
#define CPP_NE_OP 275
#define CPP_OR_OP 276
#define CPP_INC_OP 277
#define CPP_STRCONSTANT 278
#define CPP_RIGHT_ASSIGN 280
#define CPP_LEFT_ASSIGN 281
#define CPP_AND_ASSIGN 282
#define CPP_OR_ASSIGN 283
#define CPP_XOR_ASSIGN 284
#define CPP_LEFT_BRACKET 285
#define CPP_RIGHT_BRACKET 286
#define CPP_LEFT_BRACE 287
#define CPP_RIGHT_BRACE 288
#define CPP_UINTCONSTANT 289
#define CPP_DOUBLECONSTANT 290
#define CPP_TOKEN_PASTE 291
#define CPP_FIRST_USER_TOKEN_SY 292
namespace glslang {
// Multi-character tokens
enum EFixedAtoms {
PpAtomMaxSingle = 256, // single character tokens get their own char value as their token, skip them
// Operators
PpAtomAdd,
PpAtomSub,
PpAtomMul,
PpAtomDiv,
PpAtomMod,
PpAtomRight,
PpAtomLeft,
PpAtomRightAssign,
PpAtomLeftAssign,
PpAtomAndAssign,
PpAtomOrAssign,
PpAtomXorAssign,
PpAtomAnd,
PpAtomOr,
PpAtomXor,
PpAtomEQ,
PpAtomNE,
PpAtomGE,
PpAtomLE,
PpAtomDecrement,
PpAtomIncrement,
PpAtomPaste,
// Constants
PpAtomConstInt,
PpAtomConstUint,
PpAtomConstFloat,
PpAtomConstDouble,
PpAtomConstString,
// Indentifiers
PpAtomIdentifier,
// preprocessor "keywords"
PpAtomDefine,
PpAtomDefined,
PpAtomUndef,
PpAtomIf,
PpAtomIfdef,
PpAtomIfndef,
PpAtomElse,
PpAtomElif,
PpAtomEndif,
PpAtomLine,
PpAtomPragma,
PpAtomError,
// #version ...
PpAtomVersion,
PpAtomCore,
PpAtomCompatibility,
PpAtomEs,
// #extension
PpAtomExtension,
// __LINE__, __FILE__, __VERSION__
PpAtomLineMacro,
PpAtomFileMacro,
PpAtomVersionMacro,
PpAtomLast,
};
} // end namespace glslang
#endif /* not PARSER_H */
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment