Fix issue #708: token pasting within macro argument expansion.

Test/baseResults/tokenPaste.vert.out

@@ -63,6 +63,8 @@ ERROR: node is still EOpNull!
 0:?       4 (const int)
 0:?     'cop' (global int)
 0:?     'dop' (global bool)
+0:?     'argPaste2' (uniform int)
+0:?     'argPaste20suff' (uniform int)
 0:?     'gl_VertexID' (gl_VertexId int VertexId)
 0:?     'gl_InstanceID' (gl_InstanceId int InstanceId)
 
@@ -107,6 +109,8 @@ ERROR: node is still EOpNull!
 0:?       4 (const int)
 0:?     'cop' (global int)
 0:?     'dop' (global bool)
+0:?     'argPaste2' (uniform int)
+0:?     'argPaste20suff' (uniform int)
 0:?     'gl_VertexID' (gl_VertexId int VertexId)
 0:?     'gl_InstanceID' (gl_InstanceId int InstanceId)
 

Test/tokenPaste.vert

@@ -68,3 +68,12 @@ void foo()
     // recovery from bad op
     bool f = e MAKE_OP(>,!) 5;
 }
+
+// arguments: should make 'uniform int argPaste2;'
+#define M_NEST(q) int q
+#define M_OUTER(p) M_NEST(p##2)
+uniform M_OUTER(argPaste);
+// should make 'uniform int argPaste20suff;'
+#define M_NEST2(q) int q ## suff
+#define M_OUTER2(p) M_NEST2(p ## 20)
+uniform M_OUTER2(argPaste);

glslang/Include/revision.h

@@ -2,5 +2,5 @@
 // For the version, it uses the latest git tag followed by the number of commits.
 // For the date, it uses the current date (when then script is run).
 
-#define GLSLANG_REVISION "Overload400-PrecQual.1819"
-#define GLSLANG_DATE "08-Feb-2017"
+#define GLSLANG_REVISION "Overload400-PrecQual.1820"
+#define GLSLANG_DATE "10-Feb-2017"

glslang/MachineIndependent/preprocessor/Pp.cpp

@@ -979,25 +979,13 @@ int TPpContext::scanHeaderName(TPpToken* ppToken, char delimit)
 // Returns nullptr if no expanded argument is created.
 TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream& arg, TPpToken* ppToken, bool newLineOkay)
 {
-    // pre-check, to see if anything in the argument needs to be expanded,
-    // to see if we can kick out early
-    int token;
-    RewindTokenStream(arg);
-    do {
-        token = ReadToken(arg, ppToken);
-        if (token == PpAtomIdentifier && lookupMacroDef(atomStrings.getAtom(ppToken->name)) != nullptr)
-            break;
-    } while (token != EndOfInput);
-
-    // if nothing needs to be expanded, kick out early
-    if (token == EndOfInput)
-        return nullptr;
-
     // expand the argument
     TokenStream* expandedArg = new TokenStream;
     pushInput(new tMarkerInput(this));
     pushTokenStreamInput(arg);
+    int token;
     while ((token = scanToken(ppToken)) != tMarkerInput::marker && token != EndOfInput) {
+        token = tokenPaste(token, *ppToken);
         if (token == PpAtomIdentifier && MacroExpand(ppToken, false, newLineOkay) != 0)
             continue;
         RecordToken(*expandedArg, token, ppToken);
@@ -1263,7 +1251,7 @@ int TPpContext::MacroExpand(TPpToken* ppToken, bool expandUndef, bool newLineOka
         }
 
         // We need both expanded and non-expanded forms of the argument, for whether or
-        // not token pasting is in play.
+        // not token pasting will be applied later when the argument is consumed next to ##.
         for (size_t i = 0; i < in->mac->args.size(); i++)
             in->expandedArgs[i] = PrescanMacroArg(*in->args[i], ppToken, newLineOkay);
     }

glslang/MachineIndependent/preprocessor/PpTokens.cpp

@@ -275,19 +275,37 @@ int TPpContext::tTokenInput::scan(TPpToken* ppToken)
     return pp->ReadToken(*tokens, ppToken);
 }
 
-// We are pasting if the entire macro is preceding a pasting operator
-// (lastTokenPastes) and we are also on the last token.
+// We are pasting if
+//   1. we are preceding a pasting operator within this stream
+// or
+//   2. the entire macro is preceding a pasting operator (lastTokenPastes)
+//      and we are also on the last token
 bool TPpContext::tTokenInput::peekPasting()
 {
+    // 1. preceding ##?
+
+    size_t savePos = tokens->current;
+    int byte;
+    // skip white space
+    do {
+        byte = pp->lReadByte(*tokens);
+    } while (byte == ' ');
+    bool pasting = (byte == ((PpAtomPaste & 0x7f) + 0x80));
+    tokens->current = savePos;
+    if (pasting)
+        return true;
+
+    // 2. last token and we've been told after this there will be a ##
+
     if (! lastTokenPastes)
         return false;
-    // Getting here means the last token will be pasted.
+    // Getting here means the last token will be pasted, after this
 
     // Are we at the last non-whitespace token?
-    size_t savePos = tokens->current;
+    savePos = tokens->current;
     bool moreTokens = false;
     do {
-        int byte = pp->lReadByte(*tokens);
+        byte = pp->lReadByte(*tokens);
         if (byte == EndOfInput)
             break;
         if (byte != ' ') {