glsl/glcpp: Correctly parse directives with intervening comments

It's legal (though highly bizarre) for a pre-processor directive to look like
this:

	#  /* why? */ define FOO bar

This behavior comes about since the specification defines separate logical
phases in a precise order, and comment-removal occurs in a phase before the
identification of directives.

Our implementation does not use an actual separate phase for comment removal,
so some extra care is necessary to correctly parse this. What we want is for
'#' to introduce a directive iff it is the first token on a line, (ignoring
whitespace and comments). Previously, we had a lexical rule that worked only
for whitespace (not comments) with the following regular expression to find a
directive-introducing '#' at the beginning of a line:

	HASH		^{HSPACE}*#{HSPACE}*

In this commit, we switch to instead use a simple literal match of '#' to
return a HASH_TOKEN token and add a new <HASH> start condition for whenever
the HASH_TOKEN is the first non-space token of a line. This requires the
addition of the new bit of state: first_non_space_token_this_line.

This approach has a couple of implications on the glcpp parser:

	1. The parser now sees two separate tokens, (such as HASH_TOKEN and
	   HASH_DEFINE) where it previously saw one token (HASH_DEFINE) for
	   the sequence "#define". This is a straightforward change throughout
	   the grammar.

	2. The parser may now see a SPACE token before the HASH_TOKEN token of
	   a directive. Previously the lexical regular expression for {HASH}
	   would eat up the space and there would be no SPACE token.

This second implication is a bit of a nuisance for the parser. It causes a
SPACE token to appear in a production of the grammar with the following two
definitions of a control_line:

	control_line
	SPACE control_line

This is really ugly, since normally a space would simply be a token
separator, so it wouldn't appear in the tokens of a production. This leads to
a further problem with interleaved spaces and comments:

	/* ... */    /* ... */ #define /* ..*/

For this, we must not return several consecutive SPACE tokens, or else we would need an arbitrary number of new productions:

	SPACE SPACE control_line
	SPACE SPACE SPACE control_line
	ad nauseam

To avoid this problem, in this commit we also change the lexer to emit only a
single SPACE token for any series of consecutive spaces, (whether from actual
whitespace or comments). For this compression, we add a new bit of parser
state: last_token_was_space. And we also update the expected results of all
necessary test cases for the new compression of space tokens.

Fortunately, the compression of spaces should not lead to any semantic changes
in terms of what the eventual GLSL compiler sees.

So there's a lot happening in this commit, (particularly for such a tiny
feature). But fortunately, the lexer itself is looking cleaner than ever. The
only ugly bit is all the state updating, but it is at least isolated to a
single shared function.

Of course, a new "make check" test is added for the new feature, (directives
with comments and whitespace interleaved in many combinations).

And this commit fixes the following Khronos GLES3 CTS tests:

	function_definition_with_comments_vertex
	function_definition_with_comments_fragment

Reviewed-by: Jordan Justen <[email protected]>

16 files changed, 242 insertions, 110 deletions

diff --git a/src/glsl/glcpp/glcpp-lex.l b/src/glsl/glcpp/glcpp-lex.l
index 5a5bbe1886a..60bc0800b2d 100644
--- a/src/glsl/glcpp/glcpp-lex.l
+++ b/src/glsl/glcpp/glcpp-lex.l
@@ -52,7 +52,7 @@ void glcpp_set_column (int  column_no , yyscan_t yyscanner);
 		yylloc->last_column = yycolumn + 1;			\
 		parser->has_new_line_number = 0;			\
 		parser->has_new_source_number = 0;			\
- } while(0);
+	} while(0);
 
 #define YY_USER_INIT			\
 	do {				\
@@ -85,13 +85,10 @@ void glcpp_set_column (int  column_no , yyscan_t yyscanner);
  * of RETURN_TOKEN that performs a string copy of yytext before the
  * return.
  */
-#define RETURN_TOKEN_NEVER_SKIP(token)				\
-	do {							\
-		if (token == NEWLINE)				\
-			parser->last_token_was_newline = 1;	\
-		else						\
-			parser->last_token_was_newline = 0;	\
-		return (token);					\
+#define RETURN_TOKEN_NEVER_SKIP(token)					\
+	do {								\
+		if (glcpp_lex_update_state_per_token (parser, token))	\
+			return token;					\
 	} while (0)
 
 #define RETURN_TOKEN(token)						\
@@ -109,6 +106,53 @@ void glcpp_set_column (int  column_no , yyscan_t yyscanner);
 		}							\
 	} while(0)
 
+
+/* Update all state necessary for each token being returned.
+ *
+ * Here we'll be tracking newlines and spaces so that the lexer can
+ * alter its behavior as necessary, (for example, '#' has special
+ * significance if it is the first non-whitespace, non-comment token
+ * in a line, but does not otherwise).
+ *
+ * NOTE: If this function returns FALSE, then no token should be
+ * returned at all. This is used to suprress duplicate SPACE tokens.
+ */
+static int
+glcpp_lex_update_state_per_token (glcpp_parser_t *parser, int token)
+{
+	/* After the first non-space token in a line, we won't
+	 * allow any '#' to introduce a directive. */
+	if (token == NEWLINE) {
+		parser->first_non_space_token_this_line = 1;
+	} else if (token != SPACE) {
+		parser->first_non_space_token_this_line = 0;
+	}
+
+	/* Track newlines just to know whether a newline needs
+	 * to be inserted if end-of-file comes early. */
+	if (token == NEWLINE) {
+		parser->last_token_was_newline = 1;
+	} else {
+		parser->last_token_was_newline = 0;
+	}
+
+	/* Track spaces to avoid emitting multiple SPACE
+	 * tokens in a row. */
+	if (token == SPACE) {
+		if (! parser->last_token_was_space) {
+			parser->last_token_was_space = 1;
+			return 1;
+		} else {
+			parser->last_token_was_space = 1;
+			return 0;
+		}
+	} else {
+		parser->last_token_was_space = 0;
+		return 1;
+	}
+}
+
+
 %}
 
 %option bison-bridge bison-locations reentrant noyywrap
@@ -117,13 +161,13 @@ void glcpp_set_column (int  column_no , yyscan_t yyscanner);
 %option stack
 %option never-interactive
 
-%x DONE COMMENT UNREACHABLE DEFINE NEWLINE_CATCHUP
+%x DONE COMMENT HASH UNREACHABLE DEFINE NEWLINE_CATCHUP
 
 SPACE		[[:space:]]
 NONSPACE	[^[:space:]]
 NEWLINE		[\n]
 HSPACE		[ \t]
-HASH		^{HSPACE}*#{HSPACE}*
+HASH		#
 IDENTIFIER	[_a-zA-Z][_a-zA-Z0-9]*
 PP_NUMBER	[.]?[0-9]([._a-zA-Z0-9]|[eEpP][-+])*
 PUNCTUATION	[][(){}.&*~!/%<>^|;,=+-]
@@ -160,7 +204,7 @@ HEXADECIMAL_INTEGER	0[xX][0-9a-fA-F]+[uU]?
 			parser->commented_newlines--;
 		if (parser->commented_newlines == 0)
 			BEGIN INITIAL;
-		RETURN_TOKEN (NEWLINE);
+		RETURN_TOKEN_NEVER_SKIP (NEWLINE);
 	}
 
 	/* Set up the parser->skipping bit here before doing any lexing.
@@ -206,77 +250,103 @@ HEXADECIMAL_INTEGER	0[xX][0-9a-fA-F]+[uU]?
 }
 
 	/* Multi-line comments */
-<DEFINE,INITIAL>"/*"                    { yy_push_state(COMMENT, yyscanner); }
+<DEFINE,HASH,INITIAL>"/*"                    { yy_push_state(COMMENT, yyscanner); }
 <COMMENT>[^*\n]*
 <COMMENT>[^*\n]*\n      { yylineno++; yycolumn = 0; parser->commented_newlines++; }
 <COMMENT>"*"+[^*/\n]*
 <COMMENT>"*"+[^*/\n]*\n { yylineno++; yycolumn = 0; parser->commented_newlines++; }
 <COMMENT>"*"+"/"        {
 	yy_pop_state(yyscanner);
-	if (yyextra->space_tokens)
+	/* In the <HASH> start condition, we don't want any SPACE token. */
+	if (yyextra->space_tokens && YY_START != HASH)
 		RETURN_TOKEN (SPACE);
 }
 
-{HASH}version{HSPACE}+ {
+{HASH} {
+
+	/* If the '#' is the first non-whitespace, non-comment token on this
+	 * line, then it introduces a directive, switch to the <HASH> start
+	 * condition.
+	 *
+	 * Otherwise, this is just punctuation, so return the HASH_TOKEN
+         * token. */
+	if (parser->first_non_space_token_this_line) {
+		BEGIN HASH;
+	}
+
+	RETURN_TOKEN_NEVER_SKIP (HASH_TOKEN);
+}
+
+<HASH>version{HSPACE}+ {
+	BEGIN INITIAL;
 	yyextra->space_tokens = 0;
 	RETURN_STRING_TOKEN (HASH_VERSION);
 }
 
 	/* glcpp doesn't handle #extension, #version, or #pragma directives.
 	 * Simply pass them through to the main compiler's lexer/parser. */
-{HASH}(extension|pragma)[^\n]* {
+<HASH>(extension|pragma)[^\n]* {
+	BEGIN INITIAL;
 	yylineno++;
 	yycolumn = 0;
-	RETURN_STRING_TOKEN (OTHER);
+	RETURN_STRING_TOKEN (HASH_PRAGMA);
 }
 
-{HASH}line{HSPACE}+ {
+<HASH>line{HSPACE}+ {
+	BEGIN INITIAL;
 	RETURN_TOKEN (HASH_LINE);
 }
 
+<HASH>\n {
+	BEGIN INITIAL;
+	RETURN_TOKEN_NEVER_SKIP (NEWLINE);
+}
+
 	/* For the pre-processor directives, we return these tokens
 	 * even when we are otherwise skipping. */
-{HASH}ifdef {
+<HASH>ifdef {
+	BEGIN INITIAL;
 	yyextra->lexing_directive = 1;
 	yyextra->space_tokens = 0;
 	RETURN_TOKEN_NEVER_SKIP (HASH_IFDEF);
 }
 
-{HASH}ifndef {
+<HASH>ifndef {
+	BEGIN INITIAL;
 	yyextra->lexing_directive = 1;
 	yyextra->space_tokens = 0;
 	RETURN_TOKEN_NEVER_SKIP (HASH_IFNDEF);
 }
 
-{HASH}if/[^_a-zA-Z0-9] {
+<HASH>if/[^_a-zA-Z0-9] {
+	BEGIN INITIAL;
 	yyextra->lexing_directive = 1;
 	yyextra->space_tokens = 0;
 	RETURN_TOKEN_NEVER_SKIP (HASH_IF);
 }
 
-{HASH}elif/[^_a-zA-Z0-9] {
+<HASH>elif/[^_a-zA-Z0-9] {
+	BEGIN INITIAL;
 	yyextra->lexing_directive = 1;
 	yyextra->space_tokens = 0;
 	RETURN_TOKEN_NEVER_SKIP (HASH_ELIF);
 }
 
-{HASH}else {
+<HASH>else {
+	BEGIN INITIAL;
 	yyextra->space_tokens = 0;
 	RETURN_TOKEN_NEVER_SKIP (HASH_ELSE);
 }
 
-{HASH}endif {
+<HASH>endif {
+	BEGIN INITIAL;
 	yyextra->space_tokens = 0;
 	RETURN_TOKEN_NEVER_SKIP (HASH_ENDIF);
 }
 
-{HASH}error.* {
-	if (! parser->skipping) {
-		char *p;
-		for (p = yytext; !isalpha(p[0]); p++); /* skip "  #   " */
-		p += 5; /* skip "error" */
-		glcpp_error(yylloc, yyextra, "#error%s", p);
-	}
+<HASH>error.* {
+	BEGIN INITIAL;
+	RETURN_STRING_TOKEN (HASH_ERROR);
 }
 
 	/* After we see a "#define" we enter the <DEFINE> start state
@@ -297,7 +367,7 @@ HEXADECIMAL_INTEGER	0[xX][0-9a-fA-F]+[uU]?
 	 *	* Anything else, (not an identifier, not a comment,
 	 *	  and not whitespace). This will generate an error.
 	 */
-{HASH}define{HSPACE}+ {
+<HASH>define{HSPACE}+ {
 	if (! parser->skipping) {
 		BEGIN DEFINE;
 		yyextra->space_tokens = 0;
@@ -305,6 +375,24 @@ HEXADECIMAL_INTEGER	0[xX][0-9a-fA-F]+[uU]?
 	}
 }
 
+<HASH>undef {
+	BEGIN INITIAL;
+	yyextra->space_tokens = 0;
+	RETURN_TOKEN (HASH_UNDEF);
+}
+
+<HASH>{HSPACE}+ {
+	/* Nothing to do here. Importantly, don't leave the <HASH>
+	 * start condition, since it's legal to have space between the
+	 * '#' and the directive.. */
+}
+
+	/* This will catch any non-directive garbage after a HASH */
+<HASH>{NONSPACE} {
+	BEGIN INITIAL;
+	RETURN_TOKEN (HASH_GARBAGE);
+}
+
 	/* An identifier immediately followed by '(' */
 <DEFINE>{IDENTIFIER}/"(" {
 	BEGIN INITIAL;
@@ -337,16 +425,6 @@ HEXADECIMAL_INTEGER	0[xX][0-9a-fA-F]+[uU]?
 	RETURN_STRING_TOKEN (INTEGER_STRING);
 }
 
-{HASH}undef {
-	yyextra->space_tokens = 0;
-	RETURN_TOKEN (HASH_UNDEF);
-}
-
-{HASH} {
-	yyextra->space_tokens = 0;
-	RETURN_TOKEN (HASH_TOKEN);
-}
-
 {DECIMAL_INTEGER} {
 	RETURN_STRING_TOKEN (INTEGER_STRING);
 }
@@ -438,7 +516,7 @@ HEXADECIMAL_INTEGER	0[xX][0-9a-fA-F]+[uU]?
 	RETURN_TOKEN_NEVER_SKIP (NEWLINE);
 }
 
-<INITIAL,COMMENT,DEFINE><<EOF>> {
+<INITIAL,COMMENT,DEFINE,HASH><<EOF>> {
 	if (YY_START == COMMENT)
 		glcpp_error(yylloc, yyextra, "Unterminated comment");
 	if (YY_START == DEFINE)
diff --git a/src/glsl/glcpp/glcpp-parse.y b/src/glsl/glcpp/glcpp-parse.y
index 7454fb0112f..2952bf84d8d 100644
--- a/src/glsl/glcpp/glcpp-parse.y
+++ b/src/glsl/glcpp/glcpp-parse.y
@@ -167,11 +167,11 @@ add_builtin_define(glcpp_parser_t *parser, const char *name, int value);
 
 	/* We use HASH_TOKEN, not HASH to avoid a conflict with the <HASH>
          * start condition in the lexer. */
-%token COMMA_FINAL DEFINED ELIF_EXPANDED HASH_TOKEN HASH_DEFINE FUNC_IDENTIFIER OBJ_IDENTIFIER HASH_ELIF HASH_ELSE HASH_ENDIF HASH_IF HASH_IFDEF HASH_IFNDEF HASH_LINE HASH_UNDEF HASH_VERSION IDENTIFIER IF_EXPANDED INTEGER INTEGER_STRING LINE_EXPANDED NEWLINE OTHER PLACEHOLDER SPACE
+%token COMMA_FINAL DEFINED ELIF_EXPANDED HASH_TOKEN HASH_DEFINE FUNC_IDENTIFIER OBJ_IDENTIFIER HASH_ELIF HASH_ELSE HASH_ENDIF HASH_ERROR HASH_IF HASH_IFDEF HASH_IFNDEF HASH_LINE HASH_PRAGMA HASH_UNDEF HASH_VERSION HASH_GARBAGE IDENTIFIER IF_EXPANDED INTEGER INTEGER_STRING LINE_EXPANDED NEWLINE OTHER PLACEHOLDER SPACE
 %token PASTE
 %type <ival> INTEGER operator SPACE integer_constant
 %type <expression_value> expression
-%type <str> IDENTIFIER FUNC_IDENTIFIER OBJ_IDENTIFIER INTEGER_STRING OTHER
+%type <str> IDENTIFIER FUNC_IDENTIFIER OBJ_IDENTIFIER INTEGER_STRING OTHER HASH_ERROR HASH_PRAGMA
 %type <string_list> identifier_list
 %type <token> preprocessing_token conditional_token
 %type <token_list> pp_tokens replacement_list text_line conditional_tokens
@@ -197,27 +197,14 @@ input:
 ;
 
 line:
-	control_line {
-		ralloc_asprintf_rewrite_tail (&parser->output, &parser->output_length, "\n");
-	}
-|	HASH_LINE {
-		glcpp_parser_resolve_implicit_version(parser);
-	} pp_tokens NEWLINE {
-
-		if (parser->skip_stack == NULL ||
-		    parser->skip_stack->type == SKIP_NO_SKIP)
-		{
-			_glcpp_parser_expand_and_lex_from (parser,
-							   LINE_EXPANDED, $3);
-		}
-	}
+	control_line
+|	SPACE control_line
 |	text_line {
 		_glcpp_parser_print_expanded_token_list (parser, $1);
 		ralloc_asprintf_rewrite_tail (&parser->output, &parser->output_length, "\n");
 		ralloc_free ($1);
 	}
 |	expanded_line
-|	HASH_TOKEN non_directive
 ;
 
 expanded_line:
@@ -264,27 +251,45 @@ define:
 ;
 
 control_line:
-	HASH_DEFINE {
+	control_line_success {
+		ralloc_asprintf_rewrite_tail (&parser->output, &parser->output_length, "\n");
+	}
+|	control_line_error
+|	HASH_TOKEN HASH_LINE {
+		glcpp_parser_resolve_implicit_version(parser);
+	} pp_tokens NEWLINE {
+
+		if (parser->skip_stack == NULL ||
+		    parser->skip_stack->type == SKIP_NO_SKIP)
+		{
+			_glcpp_parser_expand_and_lex_from (parser,
+							   LINE_EXPANDED, $4);
+		}
+	}
+;
+
+control_line_success:
+	HASH_TOKEN HASH_DEFINE {
 		glcpp_parser_resolve_implicit_version(parser);
 	} define
-|	HASH_UNDEF {
+|	HASH_TOKEN HASH_UNDEF {
 		glcpp_parser_resolve_implicit_version(parser);
 	} IDENTIFIER NEWLINE {
 		macro_t *macro;
-		if (strcmp("__LINE__", $3) == 0
-		    || strcmp("__FILE__", $3) == 0
-		    || strcmp("__VERSION__", $3) == 0)
+		if (strcmp("__LINE__", $4) == 0
+		    || strcmp("__FILE__", $4) == 0
+		    || strcmp("__VERSION__", $4) == 0)
 			glcpp_error(& @1, parser, "Built-in (pre-defined)"
 				    " macro names can not be undefined.");
 
-		macro = hash_table_find (parser->defines, $3);
+		macro = hash_table_find (parser->defines, $4);
 		if (macro) {
-			hash_table_remove (parser->defines, $3);
+			hash_table_remove (parser->defines, $4);
 			ralloc_free (macro);
 		}
-		ralloc_free ($3);
+		ralloc_free ($4);
 	}
-|	HASH_IF {
+|	HASH_TOKEN HASH_IF {
 		glcpp_parser_resolve_implicit_version(parser);
 	} conditional_tokens NEWLINE {
 		/* Be careful to only evaluate the 'if' expression if
@@ -298,7 +303,7 @@ control_line:
 		    parser->skip_stack->type == SKIP_NO_SKIP)
 		{
 			_glcpp_parser_expand_and_lex_from (parser,
-							   IF_EXPANDED, $3);
+							   IF_EXPANDED, $4);
 		}	
 		else
 		{
@@ -306,7 +311,7 @@ control_line:
 			parser->skip_stack->type = SKIP_TO_ENDIF;
 		}
 	}
-|	HASH_IF NEWLINE {
+|	HASH_TOKEN HASH_IF NEWLINE {
 		/* #if without an expression is only an error if we
 		 *  are not skipping */
 		if (parser->skip_stack == NULL ||
@@ -316,21 +321,21 @@ control_line:
 		}	
 		_glcpp_parser_skip_stack_push_if (parser, & @1, 0);
 	}
-|	HASH_IFDEF {
+|	HASH_TOKEN HASH_IFDEF {
 		glcpp_parser_resolve_implicit_version(parser);
 	} IDENTIFIER junk NEWLINE {
-		macro_t *macro = hash_table_find (parser->defines, $3);
-		ralloc_free ($3);
+		macro_t *macro = hash_table_find (parser->defines, $4);
+		ralloc_free ($4);
 		_glcpp_parser_skip_stack_push_if (parser, & @1, macro != NULL);
 	}
-|	HASH_IFNDEF {
+|	HASH_TOKEN HASH_IFNDEF {
 		glcpp_parser_resolve_implicit_version(parser);
 	} IDENTIFIER junk NEWLINE {
-		macro_t *macro = hash_table_find (parser->defines, $3);
-		ralloc_free ($3);
-		_glcpp_parser_skip_stack_push_if (parser, & @2, macro == NULL);
+		macro_t *macro = hash_table_find (parser->defines, $4);
+		ralloc_free ($4);
+		_glcpp_parser_skip_stack_push_if (parser, & @3, macro == NULL);
 	}
-|	HASH_ELIF conditional_tokens NEWLINE {
+|	HASH_TOKEN HASH_ELIF conditional_tokens NEWLINE {
 		/* Be careful to only evaluate the 'elif' expression
 		 * if we are not skipping. When we are skipping, we
 		 * simply change to a 0-valued 'elif' on the skip
@@ -342,7 +347,7 @@ control_line:
 		    parser->skip_stack->type == SKIP_TO_ELSE)
 		{
 			_glcpp_parser_expand_and_lex_from (parser,
-							   ELIF_EXPANDED, $2);
+							   ELIF_EXPANDED, $3);
 		}
 		else if (parser->skip_stack &&
 		    parser->skip_stack->has_else)
@@ -355,7 +360,7 @@ control_line:
 							    "elif", 0);
 		}
 	}
-|	HASH_ELIF NEWLINE {
+|	HASH_TOKEN HASH_ELIF NEWLINE {
 		/* #elif without an expression is an error unless we
 		 * are skipping. */
 		if (parser->skip_stack &&
@@ -375,7 +380,7 @@ control_line:
 			glcpp_warning(& @1, parser, "ignoring illegal #elif without expression");
 		}
 	}
-|	HASH_ELSE { parser->lexing_directive = 1; } NEWLINE {
+|	HASH_TOKEN HASH_ELSE { parser->lexing_directive = 1; } NEWLINE {
 		if (parser->skip_stack &&
 		    parser->skip_stack->has_else)
 		{
@@ -388,24 +393,36 @@ control_line:
 				parser->skip_stack->has_else = true;
 		}
 	}
-|	HASH_ENDIF {
+|	HASH_TOKEN HASH_ENDIF {
 		_glcpp_parser_skip_stack_pop (parser, & @1);
 	} NEWLINE
-|	HASH_VERSION integer_constant NEWLINE {
+|	HASH_TOKEN HASH_VERSION integer_constant NEWLINE {
 		if (parser->version_resolved) {
 			glcpp_error(& @1, parser, "#version must appear on the first line");
 		}
-		_glcpp_parser_handle_version_declaration(parser, $2, NULL, true);
+		_glcpp_parser_handle_version_declaration(parser, $3, NULL, true);
 	}
-|	HASH_VERSION integer_constant IDENTIFIER NEWLINE {
+|	HASH_TOKEN HASH_VERSION integer_constant IDENTIFIER NEWLINE {
 		if (parser->version_resolved) {
 			glcpp_error(& @1, parser, "#version must appear on the first line");
 		}
-		_glcpp_parser_handle_version_declaration(parser, $2, $3, true);
+		_glcpp_parser_handle_version_declaration(parser, $3, $4, true);
 	}
 |	HASH_TOKEN NEWLINE {
 		glcpp_parser_resolve_implicit_version(parser);
 	}
+|	HASH_TOKEN HASH_PRAGMA NEWLINE {
+		ralloc_asprintf_rewrite_tail (&parser->output, &parser->output_length, "#%s", $2);
+	}
+;
+
+control_line_error:
+	HASH_TOKEN HASH_ERROR NEWLINE {
+		glcpp_error(& @1, parser, "#%s", $2);
+	}
+|	HASH_TOKEN HASH_GARBAGE pp_tokens NEWLINE  {
+		glcpp_error (& @1, parser, "Illegal non-directive after #");
+	}
 ;
 
 integer_constant:
@@ -617,12 +634,6 @@ text_line:
 |	pp_tokens NEWLINE
 ;
 
-non_directive:
-	pp_tokens NEWLINE {
-		yyerror (& @1, parser, "Invalid tokens after #");
-	}
-;
-
 replacement_list:
 	/* empty */ { $$ = NULL; }
 |	pp_tokens
@@ -1313,7 +1324,9 @@ glcpp_parser_create (const struct gl_extensions *extensions, gl_api api)
 	parser->active = NULL;
 	parser->lexing_directive = 0;
 	parser->space_tokens = 1;
-        parser->last_token_was_newline = 0;
+	parser->last_token_was_newline = 0;
+	parser->last_token_was_space = 0;
+	parser->first_non_space_token_this_line = 1;
 	parser->newline_as_space = 0;
 	parser->in_control_line = 0;
 	parser->paren_count = 0;
diff --git a/src/glsl/glcpp/glcpp.h b/src/glsl/glcpp/glcpp.h
index c5ccf18a5f7..2734f97ce25 100644
--- a/src/glsl/glcpp/glcpp.h
+++ b/src/glsl/glcpp/glcpp.h
@@ -178,6 +178,8 @@ struct glcpp_parser {
 	int lexing_directive;
 	int space_tokens;
 	int last_token_was_newline;
+	int last_token_was_space;
+	int first_non_space_token_this_line;
 	int newline_as_space;
 	int in_control_line;
 	int paren_count;
diff --git a/src/glsl/glcpp/tests/000-content-with-spaces.c.expected b/src/glsl/glcpp/tests/000-content-with-spaces.c.expected
index f49870f7aa7..00791910ed5 100644
--- a/src/glsl/glcpp/tests/000-content-with-spaces.c.expected
+++ b/src/glsl/glcpp/tests/000-content-with-spaces.c.expected
@@ -1 +1 @@
-   this is  four  tokens  with spaces
+ this is four tokens with spaces
diff --git a/src/glsl/glcpp/tests/090-hash-error.c.expected b/src/glsl/glcpp/tests/090-hash-error.c.expected
index 32954f7380e..876a6ea9cc5 100644
--- a/src/glsl/glcpp/tests/090-hash-error.c.expected
+++ b/src/glsl/glcpp/tests/090-hash-error.c.expected
@@ -1,2 +1 @@
 0:1(1): preprocessor error: #error human error
-
diff --git a/src/glsl/glcpp/tests/091-hash-line.c.expected b/src/glsl/glcpp/tests/091-hash-line.c.expected
index d6831da384e..ac9ab252f1e 100644
--- a/src/glsl/glcpp/tests/091-hash-line.c.expected
+++ b/src/glsl/glcpp/tests/091-hash-line.c.expected
@@ -3,13 +3,9 @@
 1:0(1): preprocessor error: #error source 1, line 0 error
 2:30(1): preprocessor error: #error source 2, line 30 error
 #line 0
-
 #line 25
-
 #line 0 1
-
 #line 30 2
-
 #line 45 2
 
 
diff --git a/src/glsl/glcpp/tests/100-macro-with-colon.c.expected b/src/glsl/glcpp/tests/100-macro-with-colon.c.expected
index b4360784ee0..09f1f417bdd 100644
--- a/src/glsl/glcpp/tests/100-macro-with-colon.c.expected
+++ b/src/glsl/glcpp/tests/100-macro-with-colon.c.expected
@@ -2,6 +2,6 @@
 
 
 switch (1) {
-   case 1 + 2:
-      break;
+ case 1 + 2:
+ break;
 }
diff --git a/src/glsl/glcpp/tests/108-no-space-after-hash-version.c.expected b/src/glsl/glcpp/tests/108-no-space-after-hash-version.c.expected
index 462166c9801..4f4243f947f 100644
--- a/src/glsl/glcpp/tests/108-no-space-after-hash-version.c.expected
+++ b/src/glsl/glcpp/tests/108-no-space-after-hash-version.c.expected
@@ -1 +1 @@
-0:1(2): preprocessor error: Invalid tokens after #
+0:1(1): preprocessor error: Illegal non-directive after #
diff --git a/src/glsl/glcpp/tests/109-no-space-after-hash-line.c.expected b/src/glsl/glcpp/tests/109-no-space-after-hash-line.c.expected
index 462166c9801..4f4243f947f 100644
--- a/src/glsl/glcpp/tests/109-no-space-after-hash-line.c.expected
+++ b/src/glsl/glcpp/tests/109-no-space-after-hash-line.c.expected
@@ -1 +1 @@
-0:1(2): preprocessor error: Invalid tokens after #
+0:1(1): preprocessor error: Illegal non-directive after #
diff --git a/src/glsl/glcpp/tests/110-no-space-digits-after-hash-elif.c.expected b/src/glsl/glcpp/tests/110-no-space-digits-after-hash-elif.c.expected
index 847437c9111..4d93de41dd3 100644
--- a/src/glsl/glcpp/tests/110-no-space-digits-after-hash-elif.c.expected
+++ b/src/glsl/glcpp/tests/110-no-space-digits-after-hash-elif.c.expected
@@ -1,3 +1,3 @@
-0:2(2): preprocessor error: Invalid tokens after #
+0:2(1): preprocessor error: Illegal non-directive after #
 
 
diff --git a/src/glsl/glcpp/tests/121-comment-bug-72686.c.expected b/src/glsl/glcpp/tests/121-comment-bug-72686.c.expected
index 5c484c2fe59..8cb7cb9891f 100644
--- a/src/glsl/glcpp/tests/121-comment-bug-72686.c.expected
+++ b/src/glsl/glcpp/tests/121-comment-bug-72686.c.expected
@@ -1,2 +1,2 @@
-  
+ 
 
diff --git a/src/glsl/glcpp/tests/128-space-before-hash.c.expected b/src/glsl/glcpp/tests/128-space-before-hash.c.expected
index 5d44f4161eb..9babb6fb078 100644
--- a/src/glsl/glcpp/tests/128-space-before-hash.c.expected
+++ b/src/glsl/glcpp/tests/128-space-before-hash.c.expected
@@ -1,6 +1,6 @@
-  
+ 
 #version 300
- #pragma Testing spaces before hash
+#pragma Testing spaces before hash
 
 #line 3
 
diff --git a/src/glsl/glcpp/tests/130-define-comment.c.expected b/src/glsl/glcpp/tests/130-define-comment.c.expected
index 43d399cafe4..d789e29d5a8 100644
--- a/src/glsl/glcpp/tests/130-define-comment.c.expected
+++ b/src/glsl/glcpp/tests/130-define-comment.c.expected
@@ -1,2 +1,2 @@
 
-FOO(   bar   )
+FOO( bar )
diff --git a/src/glsl/glcpp/tests/132-eof-without-newline-define.c.expected b/src/glsl/glcpp/tests/132-eof-without-newline-define.c.expected
index 57dee695714..a3ace0f3966 100644
--- a/src/glsl/glcpp/tests/132-eof-without-newline-define.c.expected
+++ b/src/glsl/glcpp/tests/132-eof-without-newline-define.c.expected
@@ -1,2 +1,2 @@
-0:1(1): preprocessor error: #define without macro name
-0:1(1): preprocessor error: syntax error, unexpected NEWLINE, expecting FUNC_IDENTIFIER or OBJ_IDENTIFIER
+0:1(2): preprocessor error: #define without macro name
+0:1(2): preprocessor error: syntax error, unexpected NEWLINE, expecting FUNC_IDENTIFIER or OBJ_IDENTIFIER
diff --git a/src/glsl/glcpp/tests/134-hash-comment-directive.c b/src/glsl/glcpp/tests/134-hash-comment-directive.c
new file mode 100644
index 00000000000..3015f0e886e
--- /dev/null
+++ b/src/glsl/glcpp/tests/134-hash-comment-directive.c
@@ -0,0 +1,22 @@
+/*...*/ # /*...*/ version 300
+ /*...*/#/*...*/  extension whatever
+ /*..*/ # /*..*/  pragma ignored
+/**/    #    /**/ line 4
+ /*...*/# /*...*/ ifdef NOT_DEFINED
+ /*...*/# /*...*/ else
+ /*..*/ #/*..*/   endif
+ /*...*/# /*...*/ ifndef ALSO_NOT_DEFINED
+ /*...*/# /*...*/ else
+ /*..*/ #/*..*/   endif
+/*...*/ # /*...*/ if 0
+ /*...*/#/*...*/  elif 1
+ /*..*/ # /*..*/  else
+ /**/   # /**/    endif
+ /*...*/# /*...*/ define FOO bar
+ /*..*/ #/*..*/   define FUNC() baz
+ /*..*/ # /*..*/  define FUNC2(a,b) b a
+FOO
+FUNC()
+FUNC2(x,y)
+
+
diff --git a/src/glsl/glcpp/tests/134-hash-comment-directive.c.expected b/src/glsl/glcpp/tests/134-hash-comment-directive.c.expected
new file mode 100644
index 00000000000..760c960cb62
--- /dev/null
+++ b/src/glsl/glcpp/tests/134-hash-comment-directive.c.expected
@@ -0,0 +1,22 @@
+#version 300
+#extension whatever
+#pragma ignored
+#line 4
+
+
+
+
+
+
+
+
+
+
+
+
+
+bar
+baz
+y x
+
+