Skip to content

G
glslang
Commit 5cdf3c5a by John Kessenich
Options

PP: Non-functional: Remove the sub-tokenization of macro stream record.

This has been a continually fragile area. Switching to a vector of real objects, instead of a linearized stream of characters, removes a bunch of code and generally makes this area more robust.
parent bce1f510
Showing with 61 additions and 161 deletions
glslang/MachineIndependent/preprocessor/PpContext.h
...@@ -243,24 +243,45 @@ public: ...@@ -243,24 +243,45 @@ public:
// From PpTokens.cpp // From PpTokens.cpp
// //
// Capture the needed parts of a token stream for macro recording/playback.
class TokenStream { class TokenStream {
public: public:
TokenStream() : current(0) { } // Manage a stream of these 'Token', which capture the relevant parts
// of a TPpToken, plus its atom.
class Token {
public:
Token(int atom, const TPpToken& ppToken) :
atom(atom),
i64val(ppToken.i64val),
name(ppToken.name) { }
int get(TPpToken& ppToken)
{
ppToken.clear();
ppToken.i64val = i64val;
snprintf(ppToken.name, sizeof(ppToken.name), "%s", name.c_str());
return atom;
}
bool isAtom(int a) { return atom == a; }
protected:
Token() {}
int atom;
long long i64val;
TString name;
};
TokenStream() : currentPos(0) { }
void putToken(int token, TPpToken* ppToken); void putToken(int token, TPpToken* ppToken);
bool peekToken(int atom) { return !atEnd() && stream[currentPos].isAtom(atom); }
int getToken(TParseContextBase&, TPpToken*); int getToken(TParseContextBase&, TPpToken*);
bool atEnd() { return current >= data.size(); } bool atEnd() { return currentPos >= stream.size(); }
bool peekTokenizedPasting(bool lastTokenPastes); bool peekTokenizedPasting(bool lastTokenPastes);
bool peekUntokenizedPasting(); bool peekUntokenizedPasting();
void reset() { current = 0; } void reset() { currentPos = 0; }
protected: protected:
void putSubtoken(char); TVector<Token> stream;
int getSubtoken(); size_t currentPos;
void ungetSubtoken();
TVector<unsigned char> data;
size_t current;
}; };
// //
......
glslang/MachineIndependent/preprocessor/PpTokens.cpp
...@@ -99,151 +99,33 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ...@@ -99,151 +99,33 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
namespace glslang { namespace glslang {
// Add a token (including backing string) to the end of a macro
namespace { // token stream, for later playback.
// When recording (and playing back) should the backing name string
// be saved (restored)?
bool SaveName(int atom)
{
switch (atom) {
case PpAtomIdentifier:
case PpAtomConstString:
case PpAtomConstInt:
case PpAtomConstUint:
case PpAtomConstInt64:
case PpAtomConstUint64:
#ifdef AMD_EXTENSIONS
case PpAtomConstInt16:
case PpAtomConstUint16:
#endif
case PpAtomConstFloat:
case PpAtomConstDouble:
case PpAtomConstFloat16:
return true;
default:
return false;
}
}
// When recording (and playing back) should the numeric value
// be saved (restored)?
bool SaveValue(int atom)
{
switch (atom) {
case PpAtomConstInt:
case PpAtomConstUint:
case PpAtomConstInt64:
case PpAtomConstUint64:
#ifdef AMD_EXTENSIONS
case PpAtomConstInt16:
case PpAtomConstUint16:
#endif
case PpAtomConstFloat:
case PpAtomConstDouble:
case PpAtomConstFloat16:
return true;
default:
return false;
}
}
}
// push onto back of stream
void TPpContext::TokenStream::putSubtoken(char subtoken)
{
data.push_back(static_cast<unsigned char>(subtoken));
}
// get the next token in stream
int TPpContext::TokenStream::getSubtoken()
{
if (current < data.size())
return data[current++];
else
return EndOfInput;
}
// back up one position in the stream
void TPpContext::TokenStream::ungetSubtoken()
{
if (current > 0)
--current;
}
// Add a complete token (including backing string) to the end of a list
// for later playback.
void TPpContext::TokenStream::putToken(int atom, TPpToken* ppToken) void TPpContext::TokenStream::putToken(int atom, TPpToken* ppToken)
{ {
// save the atom TokenStream::Token streamToken(atom, *ppToken);
assert((atom & ~0xff) == 0); stream.push_back(streamToken);
putSubtoken(static_cast<char>(atom));
// save the backing name string
if (SaveName(atom)) {
const char* s = ppToken->name;
while (*s)
putSubtoken(*s++);
putSubtoken(0);
}
// save the numeric value
if (SaveValue(atom)) {
const char* n = reinterpret_cast<const char*>(&ppToken->i64val);
for (size_t i = 0; i < sizeof(ppToken->i64val); ++i)
putSubtoken(*n++);
}
} }
// Read the next token from a token stream. // Read the next token from a macro token stream.
// (Not the source stream, but a stream used to hold a tokenized macro).
int TPpContext::TokenStream::getToken(TParseContextBase& parseContext, TPpToken *ppToken) int TPpContext::TokenStream::getToken(TParseContextBase& parseContext, TPpToken *ppToken)
{ {
// get the atom if (atEnd())
int atom = getSubtoken(); return EndOfInput;
if (atom == EndOfInput)
return atom;
// init the token int atom = stream[currentPos++].get(*ppToken);
ppToken->clear();
ppToken->loc = parseContext.getCurrentLoc(); ppToken->loc = parseContext.getCurrentLoc();
// get the backing name string
if (SaveName(atom)) {
int ch = getSubtoken();
int len = 0;
while (ch != 0 && ch != EndOfInput) {
if (len < MaxTokenLength) {
ppToken->name[len] = (char)ch;
len++;
ch = getSubtoken();
} else {
parseContext.error(ppToken->loc, "token too long", "", "");
break;
}
}
ppToken->name[len] = 0;
}
// Check for ##, unless the current # is the last character // Check for ##, unless the current # is the last character
if (atom == '#') { if (atom == '#') {
if (current < data.size()) { if (peekToken('#')) {
if (getSubtoken() == '#') { parseContext.requireProfile(ppToken->loc, ~EEsProfile, "token pasting (##)");
parseContext.requireProfile(ppToken->loc, ~EEsProfile, "token pasting (##)"); parseContext.profileRequires(ppToken->loc, ~EEsProfile, 130, 0, "token pasting (##)");
parseContext.profileRequires(ppToken->loc, ~EEsProfile, 130, 0, "token pasting (##)"); currentPos++;
atom = PpAtomPaste; atom = PpAtomPaste;
} else
ungetSubtoken();
} }
} }
// get the numeric value
if (SaveValue(atom)) {
char* n = reinterpret_cast<char*>(&ppToken->i64val);
for (size_t i = 0; i < sizeof(ppToken->i64val); ++i)
*n++ = (char)getSubtoken();
}
return atom; return atom;
} }
...@@ -256,15 +138,14 @@ bool TPpContext::TokenStream::peekTokenizedPasting(bool lastTokenPastes) ...@@ -256,15 +138,14 @@ bool TPpContext::TokenStream::peekTokenizedPasting(bool lastTokenPastes)
{ {
// 1. preceding ##? // 1. preceding ##?
size_t savePos = current; size_t savePos = currentPos;
int subtoken;
// skip white space // skip white space
do { while (peekToken(' '))
subtoken = getSubtoken(); ++currentPos;
} while (subtoken == ' '); if (peekToken(PpAtomPaste)) {
current = savePos; currentPos = savePos;
if (subtoken == PpAtomPaste)
return true; return true;
}
// 2. last token and we've been told after this there will be a ## // 2. last token and we've been told after this there will be a ##
...@@ -273,18 +154,18 @@ bool TPpContext::TokenStream::peekTokenizedPasting(bool lastTokenPastes) ...@@ -273,18 +154,18 @@ bool TPpContext::TokenStream::peekTokenizedPasting(bool lastTokenPastes)
// Getting here means the last token will be pasted, after this // Getting here means the last token will be pasted, after this
// Are we at the last non-whitespace token? // Are we at the last non-whitespace token?
savePos = current; savePos = currentPos;
bool moreTokens = false; bool moreTokens = false;
do { do {
subtoken = getSubtoken(); if (atEnd())
if (subtoken == EndOfInput)
break; break;
if (subtoken != ' ') { if (!peekToken(' ')) {
moreTokens = true; moreTokens = true;
break; break;
} }
++currentPos;
} while (true); } while (true);
current = savePos; currentPos = savePos;
return !moreTokens; return !moreTokens;
} }
...@@ -293,23 +174,21 @@ bool TPpContext::TokenStream::peekTokenizedPasting(bool lastTokenPastes) ...@@ -293,23 +174,21 @@ bool TPpContext::TokenStream::peekTokenizedPasting(bool lastTokenPastes)
bool TPpContext::TokenStream::peekUntokenizedPasting() bool TPpContext::TokenStream::peekUntokenizedPasting()
{ {
// don't return early, have to restore this // don't return early, have to restore this
size_t savePos = current; size_t savePos = currentPos;
// skip white-space // skip white-space
int subtoken; while (peekToken(' '))
do { ++currentPos;
subtoken = getSubtoken();
} while (subtoken == ' ');
// check for ## // check for ##
bool pasting = false; bool pasting = false;
if (subtoken == '#') { if (peekToken('#')) {
subtoken = getSubtoken(); ++currentPos;
if (subtoken == '#') if (peekToken('#'))
pasting = true; pasting = true;
} }
current = savePos; currentPos = savePos;
return pasting; return pasting;
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment