glsl/pp: Do purification and tokenisation in a single step.

6 files changed, 561 insertions, 307 deletions

diff --git a/src/glsl/pp/sl_pp_context.c b/src/glsl/pp/sl_pp_context.c
index 8ce189d955c..134588d9066 100644
--- a/src/glsl/pp/sl_pp_context.c
+++ b/src/glsl/pp/sl_pp_context.c
@@ -46,6 +46,12 @@ sl_pp_context_create(void)
       return NULL;
    }
 
+   context->getc_buf = malloc(64 * sizeof(char));
+   if (!context->getc_buf) {
+      sl_pp_context_destroy(context);
+      return NULL;
+   }
+
    context->macro_tail = &context->macro;
    context->if_ptr = SL_PP_MAX_IF_NESTING;
    context->if_value = 1;
@@ -62,6 +68,7 @@ sl_pp_context_destroy(struct sl_pp_context *context)
    if (context) {
       free(context->cstr_pool);
       sl_pp_macro_free(context->macro);
+      free(context->getc_buf);
       free(context);
    }
 }
diff --git a/src/glsl/pp/sl_pp_context.h b/src/glsl/pp/sl_pp_context.h
index 6b8cc2f960d..569a2d735b9 100644
--- a/src/glsl/pp/sl_pp_context.h
+++ b/src/glsl/pp/sl_pp_context.h
@@ -30,6 +30,7 @@
 
 #include "sl_pp_dict.h"
 #include "sl_pp_macro.h"
+#include "sl_pp_purify.h"
 
 
 #define SL_PP_MAX_IF_NESTING  64
@@ -53,10 +54,12 @@ struct sl_pp_context {
 
    unsigned int line;
    unsigned int file;
-};
 
-int
-sl_pp_context_add_unique_str(struct sl_pp_context *context,
-                             const char *str);
+   struct sl_pp_purify_state pure;
+
+   char *getc_buf;
+   unsigned int getc_buf_size;
+   unsigned int getc_buf_capacity;
+};
 
 #endif /* SL_PP_CONTEXT_H */
diff --git a/src/glsl/pp/sl_pp_dict.c b/src/glsl/pp/sl_pp_dict.c
index 82fb9127b50..2dd77a69e90 100644
--- a/src/glsl/pp/sl_pp_dict.c
+++ b/src/glsl/pp/sl_pp_dict.c
@@ -25,6 +25,7 @@
  * 
  **************************************************************************/
 
+#include "sl_pp_public.h"
 #include "sl_pp_context.h"
 #include "sl_pp_dict.h"
 
diff --git a/src/glsl/pp/sl_pp_public.h b/src/glsl/pp/sl_pp_public.h
index b1d92d02a7a..8317c7e378a 100644
--- a/src/glsl/pp/sl_pp_public.h
+++ b/src/glsl/pp/sl_pp_public.h
@@ -45,6 +45,10 @@ sl_pp_context_destroy(struct sl_pp_context *context);
 const char *
 sl_pp_context_error_message(const struct sl_pp_context *context);
 
+int
+sl_pp_context_add_unique_str(struct sl_pp_context *context,
+                             const char *str);
+
 const char *
 sl_pp_context_cstr(const struct sl_pp_context *context,
                    int offset);
diff --git a/src/glsl/pp/sl_pp_token.c b/src/glsl/pp/sl_pp_token.c
index f232dafc682..03f2f09cfd6 100644
--- a/src/glsl/pp/sl_pp_token.c
+++ b/src/glsl/pp/sl_pp_token.c
@@ -25,12 +25,106 @@
  * 
  **************************************************************************/
 
+#include <assert.h>
 #include <stdlib.h>
 #include <string.h>
+#include "sl_pp_public.h"
 #include "sl_pp_context.h"
 #include "sl_pp_token.h"
 
 
+#define PURE_ERROR 256
+
+int
+_pure_getc(struct sl_pp_context *context)
+{
+   char c;
+   unsigned int current_line;
+
+   if (context->getc_buf_size) {
+      return context->getc_buf[--context->getc_buf_size];
+   }
+
+   if (sl_pp_purify_getc(&context->pure, &c, &current_line, context->error_msg, sizeof(context->error_msg)) == 0) {
+      return PURE_ERROR;
+   }
+   return c;
+}
+
+
+void
+_pure_ungetc(struct sl_pp_context *context,
+             int c)
+{
+   assert(c != PURE_ERROR);
+
+   if (context->getc_buf_size == context->getc_buf_capacity) {
+      context->getc_buf_capacity += 64;
+      context->getc_buf = realloc(context->getc_buf, context->getc_buf_capacity * sizeof(char));
+      assert(context->getc_buf);                            
+   }
+
+   context->getc_buf[context->getc_buf_size++] = (char)c;
+}
+
+
+struct lookahead_state {
+   char buf[256];
+   unsigned int pos;
+   struct sl_pp_context *context;
+};
+
+
+static void
+_lookahead_init(struct lookahead_state *lookahead,
+                struct sl_pp_context *context)
+{
+   lookahead->pos = 0;
+   lookahead->context = context;
+}
+
+
+static unsigned int
+_lookahead_tell(const struct lookahead_state *lookahead)
+{
+   return lookahead->pos;
+}
+
+
+static const void *
+_lookahead_buf(const struct lookahead_state *lookahead)
+{
+   return lookahead->buf;
+}
+
+
+static void
+_lookahead_revert(struct lookahead_state *lookahead,
+                  unsigned int pos)
+{
+   assert(pos <= lookahead->pos);
+
+   while (lookahead->pos > pos) {
+      _pure_ungetc(lookahead->context, lookahead->buf[--lookahead->pos]);
+   }
+}
+
+
+static int
+_lookahead_getc(struct lookahead_state *lookahead)
+{
+   int c;
+
+   assert(lookahead->pos < sizeof(lookahead->buf) / sizeof(lookahead->buf[0]));
+
+   c = _pure_getc(lookahead->context);
+   if (c != PURE_ERROR) {
+      lookahead->buf[lookahead->pos++] = (char)c;
+   }
+   return c;
+}
+
+
 static int
 _is_identifier_char(char c)
 {
@@ -40,32 +134,51 @@ _is_identifier_char(char c)
 
 static int
 _tokenise_identifier(struct sl_pp_context *context,
-                     const char **pinput,
-                     struct sl_pp_token_info *info)
+                     struct sl_pp_token_info *out)
 {
-   const char *input = *pinput;
+   int c;
    char identifier[256];   /* XXX: Remove this artifical limit. */
    unsigned int i = 0;
 
-   info->token = SL_PP_IDENTIFIER;
-   info->data.identifier = -1;
+   out->token = SL_PP_IDENTIFIER;
+   out->data.identifier = -1;
 
-   identifier[i++] = *input++;
-   while (_is_identifier_char(*input)) {
-      if (i >= sizeof(identifier) - 1) {
-         strcpy(context->error_msg, "out of memory");
+   c = _pure_getc(context);
+   if (c == PURE_ERROR) {
+      return -1;
+   }
+   identifier[i++] = (char)c;
+   for (;;) {
+      c = _pure_getc(context);
+      if (c == PURE_ERROR) {
          return -1;
       }
-      identifier[i++] = *input++;
+
+      if (_is_identifier_char((char)c)) {
+         if (i >= sizeof(identifier) / sizeof(char) - 1) {
+            strcpy(context->error_msg, "out of memory");
+            _pure_ungetc(context, c);
+            while (i) {
+               _pure_ungetc(context, identifier[--i]);
+            }
+            return -1;
+         }
+         identifier[i++] = (char)c;
+      } else {
+         _pure_ungetc(context, c);
+         break;
+      }
    }
-   identifier[i++] = '\0';
+   identifier[i] = '\0';
 
-   info->data.identifier = sl_pp_context_add_unique_str(context, identifier);
-   if (info->data.identifier == -1) {
+   out->data.identifier = sl_pp_context_add_unique_str(context, identifier);
+   if (out->data.identifier == -1) {
+      while (i) {
+         _pure_ungetc(context, identifier[--i]);
+      }
       return -1;
    }
 
-   *pinput = input;
    return 0;
 }
 
@@ -74,12 +187,18 @@ _tokenise_identifier(struct sl_pp_context *context,
  * Return the number of consecutive decimal digits in the input stream.
  */
 static unsigned int
-_parse_float_digits(const char *input)
+_parse_float_digits(struct lookahead_state *lookahead)
 {
-   unsigned int eaten = 0;
+   unsigned int eaten;
+
+   for (eaten = 0;; eaten++) {
+      unsigned int pos = _lookahead_tell(lookahead);
+      char c = _lookahead_getc(lookahead);
 
-   while (input[eaten] >= '0' && input[eaten] <= '9') {
-      eaten++;
+      if (c < '0' || c > '9') {
+         _lookahead_revert(lookahead, pos);
+         break;
+      }
    }
    return eaten;
 }
@@ -96,29 +215,33 @@ _parse_float_digits(const char *input)
  * of eaten characters from the input stream.
  */
 static unsigned int
-_parse_float_frac(const char *input)
+_parse_float_frac(struct lookahead_state *lookahead)
 {
+   unsigned int pos;
+   int c;
    unsigned int eaten;
 
-   if (input[0] == '.') {
-      eaten = _parse_float_digits(&input[1]);
+   pos = _lookahead_tell(lookahead);
+   c = _lookahead_getc(lookahead);
+   if (c == '.') {
+      eaten = _parse_float_digits(lookahead);
       if (eaten) {
          return eaten + 1;
       }
+      _lookahead_revert(lookahead, pos);
       return 0;
    }
 
-   eaten = _parse_float_digits(input);
-   if (eaten && input[eaten] == '.') {
-      unsigned int trailing;
-
-      trailing = _parse_float_digits(&input[eaten + 1]);
-      if (trailing) {
-         return eaten + trailing + 1;
+   _lookahead_revert(lookahead, pos);
+   eaten = _parse_float_digits(lookahead);
+   if (eaten) {
+      c = _lookahead_getc(lookahead);
+      if (c == '.') {
+         return eaten + 1 + _parse_float_digits(lookahead);
       }
-      return eaten + 1;
    }
 
+   _lookahead_revert(lookahead, pos);
    return 0;
 }
 
@@ -133,22 +256,31 @@ _parse_float_frac(const char *input)
  * of eaten characters from the input stream.
  */
 static unsigned int
-_parse_float_exp(const char *input)
+_parse_float_exp(struct lookahead_state *lookahead)
 {
+   unsigned int pos, pos2;
+   int c;
    unsigned int eaten, digits;
 
-   if (input[0] != 'e' && input[0] != 'E') {
+   pos = _lookahead_tell(lookahead);
+   c = _lookahead_getc(lookahead);
+   if (c != 'e' && c != 'E') {
+      _lookahead_revert(lookahead, pos);
       return 0;
    }
 
-   if (input[1] == '-' || input[1] == '+') {
+   pos2 = _lookahead_tell(lookahead);
+   c = _lookahead_getc(lookahead);
+   if (c == '-' || c == '+') {
       eaten = 2;
    } else {
+      _lookahead_revert(lookahead, pos2);
       eaten = 1;
    }
 
-   digits = _parse_float_digits(&input[eaten]);
+   digits = _parse_float_digits(lookahead);
    if (!digits) {
+      _lookahead_revert(lookahead, pos);
       return 0;
    }
 
@@ -166,86 +298,117 @@ _parse_float_exp(const char *input)
  * of eaten characters from the input stream.
  */
 static unsigned int
-_parse_float(const char *input)
+_parse_float(struct lookahead_state *lookahead)
 {
    unsigned int eaten;
 
-   eaten = _parse_float_frac(input);
+   eaten = _parse_float_frac(lookahead);
    if (eaten) {
-      unsigned int exponent;
+      unsigned int pos;
+      int c;
 
-      exponent = _parse_float_exp(&input[eaten]);
-      if (exponent) {
-         eaten += exponent;
-      }
+      eaten += _parse_float_exp(lookahead);
 
-      if (input[eaten] == 'f' || input[eaten] == 'F') {
+      pos = _lookahead_tell(lookahead);
+      c = _lookahead_getc(lookahead);
+      if (c == 'f' || c == 'F') {
          eaten++;
+      } else {
+         _lookahead_revert(lookahead, pos);
       }
 
       return eaten;
    }
 
-   eaten = _parse_float_digits(input);
+   eaten = _parse_float_digits(lookahead);
    if (eaten) {
       unsigned int exponent;
 
-      exponent = _parse_float_exp(&input[eaten]);
+      exponent = _parse_float_exp(lookahead);
       if (exponent) {
+         unsigned int pos;
+         int c;
+
          eaten += exponent;
 
-         if (input[eaten] == 'f' || input[eaten] == 'F') {
+         pos = _lookahead_tell(lookahead);
+         c = _lookahead_getc(lookahead);
+         if (c == 'f' || c == 'F') {
             eaten++;
+         } else {
+            _lookahead_revert(lookahead, pos);
          }
 
          return eaten;
       }
    }
 
+   _lookahead_revert(lookahead, 0);
    return 0;
 }
 
 
 static unsigned int
-_parse_hex(const char *input)
+_parse_hex(struct lookahead_state *lookahead)
 {
+   int c;
    unsigned int n;
 
-   if (input[0] != '0') {
+   c = _lookahead_getc(lookahead);
+   if (c != '0') {
+      _lookahead_revert(lookahead, 0);
       return 0;
    }
 
-   if (input[1] != 'x' && input[1] != 'X') {
+   c = _lookahead_getc(lookahead);
+   if (c != 'x' && c != 'X') {
+      _lookahead_revert(lookahead, 0);
       return 0;
    }
 
-   n = 2;
-   while ((input[n] >= '0' && input[n] <= '9') ||
-          (input[n] >= 'a' && input[n] <= 'f') ||
-          (input[n] >= 'A' && input[n] <= 'F')) {
-      n++;
+   for (n = 2;;) {
+      unsigned int pos = _lookahead_tell(lookahead);
+
+      c = _lookahead_getc(lookahead);
+      if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) {
+         n++;
+      } else {
+         _lookahead_revert(lookahead, pos);
+         break;
+      }
    }
 
    if (n > 2) {
       return n;
    }
 
+   _lookahead_revert(lookahead, 0);
    return 0;
 }
 
 
 static unsigned int
-_parse_oct(const char *input)
+_parse_oct(struct lookahead_state *lookahead)
 {
+   int c;
    unsigned int n;
 
-   if (input[0] != '0') {
+   c = _lookahead_getc(lookahead);
+   if (c != '0') {
+      _lookahead_revert(lookahead, 0);
       return 0;
    }
 
-   n = 1;
-   while ((input[n] >= '0' && input[n] <= '7')) {
-      n++;
+   for (n = 1;;) {
+      unsigned int pos = _lookahead_tell(lookahead);
+
+      c = _lookahead_getc(lookahead);
+      if ((c >= '0' && c <= '7')) {
+         n++;
+      } else {
+         _lookahead_revert(lookahead, pos);
+         break;
+      }
    }
 
    return n;
@@ -253,12 +416,20 @@ _parse_oct(const char *input)
 
 
 static unsigned int
-_parse_dec(const char *input)
+_parse_dec(struct lookahead_state *lookahead)
 {
    unsigned int n = 0;
 
-   while ((input[n] >= '0' && input[n] <= '9')) {
-      n++;
+   for (;;) {
+      unsigned int pos = _lookahead_tell(lookahead);
+      int c = _lookahead_getc(lookahead);
+
+      if ((c >= '0' && c <= '9')) {
+         n++;
+      } else {
+         _lookahead_revert(lookahead, pos);
+         break;
+      }
    }
 
    return n;
@@ -267,327 +438,394 @@ _parse_dec(const char *input)
 
 static int
 _tokenise_number(struct sl_pp_context *context,
-                 const char **pinput,
-                 struct sl_pp_token_info *info)
+                 struct sl_pp_token_info *out)
 {
-   const char *input = *pinput;
+   struct lookahead_state lookahead;
    unsigned int eaten;
    unsigned int is_float = 0;
+   unsigned int pos;
+   int c;
    char number[256];   /* XXX: Remove this artifical limit. */
 
-   eaten = _parse_float(input);
+   _lookahead_init(&lookahead, context);
+
+   eaten = _parse_float(&lookahead);
    if (!eaten) {
-      eaten = _parse_hex(input);
+      eaten = _parse_hex(&lookahead);
       if (!eaten) {
-         eaten = _parse_oct(input);
+         eaten = _parse_oct(&lookahead);
          if (!eaten) {
-            eaten = _parse_dec(input);
+            eaten = _parse_dec(&lookahead);
          }
       }
    } else {
       is_float = 1;
    }
 
-   if (!eaten || _is_identifier_char(input[eaten])) {
+   if (!eaten) {
       strcpy(context->error_msg, "expected a number");
       return -1;
    }
 
+   pos = _lookahead_tell(&lookahead);
+   c = _lookahead_getc(&lookahead);
+   _lookahead_revert(&lookahead, pos);
+
+   if (_is_identifier_char(c)) {
+      strcpy(context->error_msg, "expected a number");
+      _lookahead_revert(&lookahead, 0);
+      return -1;
+   }
+
    if (eaten > sizeof(number) - 1) {
       strcpy(context->error_msg, "out of memory");
+      _lookahead_revert(&lookahead, 0);
       return -1;
    }
 
-   memcpy(number, input, eaten);
+   assert(_lookahead_tell(&lookahead) == eaten);
+
+   memcpy(number, _lookahead_buf(&lookahead), eaten);
    number[eaten] = '\0';
 
    if (is_float) {
-      info->token = SL_PP_FLOAT;
-      info->data._float = sl_pp_context_add_unique_str(context, number);
-      if (info->data._float == -1) {
+      out->token = SL_PP_FLOAT;
+      out->data._float = sl_pp_context_add_unique_str(context, number);
+      if (out->data._float == -1) {
+         _lookahead_revert(&lookahead, 0);
          return -1;
       }
    } else {
-      info->token = SL_PP_UINT;
-      info->data._uint = sl_pp_context_add_unique_str(context, number);
-      if (info->data._uint == -1) {
+      out->token = SL_PP_UINT;
+      out->data._uint = sl_pp_context_add_unique_str(context, number);
+      if (out->data._uint == -1) {
+         _lookahead_revert(&lookahead, 0);
          return -1;
       }
    }
 
-   *pinput = input + eaten;
    return 0;
 }
 
 
 int
-sl_pp_tokenise(struct sl_pp_context *context,
-               const char *input,
-               struct sl_pp_token_info **output)
+sl_pp_token_get(struct sl_pp_context *context,
+                struct sl_pp_token_info *out)
 {
-   struct sl_pp_token_info *out = NULL;
-   unsigned int out_len = 0;
-   unsigned int out_max = 0;
+   int c = _pure_getc(context);
 
-   for (;;) {
-      struct sl_pp_token_info info;
+   switch (c) {
+   case ' ':
+   case '\t':
+      out->token = SL_PP_WHITESPACE;
+      break;
 
-      switch (*input) {
-      case ' ':
-      case '\t':
-         input++;
-         info.token = SL_PP_WHITESPACE;
-         break;
+   case '\n':
+      out->token = SL_PP_NEWLINE;
+      break;
 
-      case '\n':
-         input++;
-         info.token = SL_PP_NEWLINE;
-         break;
+   case '#':
+      out->token = SL_PP_HASH;
+      break;
 
-      case '#':
-         input++;
-         info.token = SL_PP_HASH;
-         break;
+   case ',':
+      out->token = SL_PP_COMMA;
+      break;
 
-      case ',':
-         input++;
-         info.token = SL_PP_COMMA;
-         break;
+   case ';':
+      out->token = SL_PP_SEMICOLON;
+      break;
 
-      case ';':
-         input++;
-         info.token = SL_PP_SEMICOLON;
-         break;
+   case '{':
+      out->token = SL_PP_LBRACE;
+      break;
 
-      case '{':
-         input++;
-         info.token = SL_PP_LBRACE;
-         break;
+   case '}':
+      out->token = SL_PP_RBRACE;
+      break;
 
-      case '}':
-         input++;
-         info.token = SL_PP_RBRACE;
-         break;
+   case '(':
+      out->token = SL_PP_LPAREN;
+      break;
 
-      case '(':
-         input++;
-         info.token = SL_PP_LPAREN;
-         break;
+   case ')':
+      out->token = SL_PP_RPAREN;
+      break;
 
-      case ')':
-         input++;
-         info.token = SL_PP_RPAREN;
-         break;
+   case '[':
+      out->token = SL_PP_LBRACKET;
+      break;
 
-      case '[':
-         input++;
-         info.token = SL_PP_LBRACKET;
-         break;
+   case ']':
+      out->token = SL_PP_RBRACKET;
+      break;
 
-      case ']':
-         input++;
-         info.token = SL_PP_RBRACKET;
-         break;
+   case '.':
+      {
+         int c2 = _pure_getc(context);
 
-      case '.':
-         if (input[1] >= '0' && input[1] <= '9') {
-            if (_tokenise_number(context, &input, &info)) {
-               free(out);
+         if (c2 == PURE_ERROR) {
+            return -1;
+         }
+         if (c2 >= '0' && c2 <= '9') {
+            _pure_ungetc(context, c2);
+            _pure_ungetc(context, c);
+            if (_tokenise_number(context, out)) {
                return -1;
             }
          } else {
-            input++;
-            info.token = SL_PP_DOT;
+            _pure_ungetc(context, c2);
+            out->token = SL_PP_DOT;
          }
-         break;
+      }
+      break;
 
-      case '+':
-         input++;
-         if (*input == '+') {
-            input++;
-            info.token = SL_PP_INCREMENT;
-         } else if (*input == '=') {
-            input++;
-            info.token = SL_PP_ADDASSIGN;
-         } else {
-            info.token = SL_PP_PLUS;
-         }
-         break;
+   case '+':
+      c = _pure_getc(context);
+      if (c == PURE_ERROR) {
+         return -1;
+      }
+      if (c == '+') {
+         out->token = SL_PP_INCREMENT;
+      } else if (c == '=') {
+         out->token = SL_PP_ADDASSIGN;
+      } else {
+         _pure_ungetc(context, c);
+         out->token = SL_PP_PLUS;
+      }
+      break;
 
-      case '-':
-         input++;
-         if (*input == '-') {
-            input++;
-            info.token = SL_PP_DECREMENT;
-         } else if (*input == '=') {
-            input++;
-            info.token = SL_PP_SUBASSIGN;
-         } else {
-            info.token = SL_PP_MINUS;
-         }
-         break;
+   case '-':
+      c = _pure_getc(context);
+      if (c == PURE_ERROR) {
+         return -1;
+      }
+      if (c == '-') {
+         out->token = SL_PP_DECREMENT;
+      } else if (c == '=') {
+         out->token = SL_PP_SUBASSIGN;
+      } else {
+         _pure_ungetc(context, c);
+         out->token = SL_PP_MINUS;
+      }
+      break;
 
-      case '~':
-         input++;
-         info.token = SL_PP_BITNOT;
-         break;
+   case '~':
+      out->token = SL_PP_BITNOT;
+      break;
 
-      case '!':
-         input++;
-         if (*input == '=') {
-            input++;
-            info.token = SL_PP_NOTEQUAL;
-         } else {
-            info.token = SL_PP_NOT;
-         }
-         break;
+   case '!':
+      c = _pure_getc(context);
+      if (c == PURE_ERROR) {
+         return -1;
+      }
+      if (c == '=') {
+         out->token = SL_PP_NOTEQUAL;
+      } else {
+         _pure_ungetc(context, c);
+         out->token = SL_PP_NOT;
+      }
+      break;
 
-      case '*':
-         input++;
-         if (*input == '=') {
-            input++;
-            info.token = SL_PP_MULASSIGN;
-         } else {
-            info.token = SL_PP_STAR;
-         }
-         break;
+   case '*':
+      c = _pure_getc(context);
+      if (c == PURE_ERROR) {
+         return -1;
+      }
+      if (c == '=') {
+         out->token = SL_PP_MULASSIGN;
+      } else {
+         _pure_ungetc(context, c);
+         out->token = SL_PP_STAR;
+      }
+      break;
 
-      case '/':
-         input++;
-         if (*input == '=') {
-            input++;
-            info.token = SL_PP_DIVASSIGN;
-         } else {
-            info.token = SL_PP_SLASH;
-         }
-         break;
+   case '/':
+      c = _pure_getc(context);
+      if (c == PURE_ERROR) {
+         return -1;
+      }
+      if (c == '=') {
+         out->token = SL_PP_DIVASSIGN;
+      } else {
+         _pure_ungetc(context, c);
+         out->token = SL_PP_SLASH;
+      }
+      break;
 
-      case '%':
-         input++;
-         if (*input == '=') {
-            input++;
-            info.token = SL_PP_MODASSIGN;
-         } else {
-            info.token = SL_PP_MODULO;
-         }
-         break;
+   case '%':
+      c = _pure_getc(context);
+      if (c == PURE_ERROR) {
+         return -1;
+      }
+      if (c == '=') {
+         out->token = SL_PP_MODASSIGN;
+      } else {
+         _pure_ungetc(context, c);
+         out->token = SL_PP_MODULO;
+      }
+      break;
 
-      case '<':
-         input++;
-         if (*input == '<') {
-            input++;
-            if (*input == '=') {
-               input++;
-               info.token = SL_PP_LSHIFTASSIGN;
-            } else {
-               info.token = SL_PP_LSHIFT;
-            }
-         } else if (*input == '=') {
-            input++;
-            info.token = SL_PP_LESSEQUAL;
-         } else {
-            info.token = SL_PP_LESS;
+   case '<':
+      c = _pure_getc(context);
+      if (c == PURE_ERROR) {
+         return -1;
+      }
+      if (c == '<') {
+         c = _pure_getc(context);
+         if (c == PURE_ERROR) {
+            return -1;
          }
-         break;
-
-      case '>':
-         input++;
-         if (*input == '>') {
-            input++;
-            if (*input == '=') {
-               input++;
-               info.token = SL_PP_RSHIFTASSIGN;
-            } else {
-               info.token = SL_PP_RSHIFT;
-            }
-         } else if (*input == '=') {
-            input++;
-            info.token = SL_PP_GREATEREQUAL;
+         if (c == '=') {
+            out->token = SL_PP_LSHIFTASSIGN;
          } else {
-            info.token = SL_PP_GREATER;
+            _pure_ungetc(context, c);
+            out->token = SL_PP_LSHIFT;
          }
-         break;
+      } else if (c == '=') {
+         out->token = SL_PP_LESSEQUAL;
+      } else {
+         _pure_ungetc(context, c);
+         out->token = SL_PP_LESS;
+      }
+      break;
 
-      case '=':
-         input++;
-         if (*input == '=') {
-            input++;
-            info.token = SL_PP_EQUAL;
-         } else {
-            info.token = SL_PP_ASSIGN;
+   case '>':
+      c = _pure_getc(context);
+      if (c == PURE_ERROR) {
+         return -1;
+      }
+      if (c == '>') {
+         c = _pure_getc(context);
+         if (c == PURE_ERROR) {
+            return -1;
          }
-         break;
-
-      case '&':
-         input++;
-         if (*input == '&') {
-            input++;
-            info.token = SL_PP_AND;
-         } else if (*input == '=') {
-            input++;
-            info.token = SL_PP_BITANDASSIGN;
+         if (c == '=') {
+            out->token = SL_PP_RSHIFTASSIGN;
          } else {
-            info.token = SL_PP_BITAND;
+            _pure_ungetc(context, c);
+            out->token = SL_PP_RSHIFT;
          }
-         break;
+      } else if (c == '=') {
+         out->token = SL_PP_GREATEREQUAL;
+      } else {
+         _pure_ungetc(context, c);
+         out->token = SL_PP_GREATER;
+      }
+      break;
 
-      case '^':
-         input++;
-         if (*input == '^') {
-            input++;
-            info.token = SL_PP_XOR;
-         } else if (*input == '=') {
-            input++;
-            info.token = SL_PP_BITXORASSIGN;
-         } else {
-            info.token = SL_PP_BITXOR;
-         }
-         break;
+   case '=':
+      c = _pure_getc(context);
+      if (c == PURE_ERROR) {
+         return -1;
+      }
+      if (c == '=') {
+         out->token = SL_PP_EQUAL;
+      } else {
+         _pure_ungetc(context, c);
+         out->token = SL_PP_ASSIGN;
+      }
+      break;
 
-      case '|':
-         input++;
-         if (*input == '|') {
-            input++;
-            info.token = SL_PP_OR;
-         } else if (*input == '=') {
-            input++;
-            info.token = SL_PP_BITORASSIGN;
-         } else {
-            info.token = SL_PP_BITOR;
-         }
-         break;
+   case '&':
+      c = _pure_getc(context);
+      if (c == PURE_ERROR) {
+         return -1;
+      }
+      if (c == '&') {
+         out->token = SL_PP_AND;
+      } else if (c == '=') {
+         out->token = SL_PP_BITANDASSIGN;
+      } else {
+         _pure_ungetc(context, c);
+         out->token = SL_PP_BITAND;
+      }
+      break;
 
-      case '?':
-         input++;
-         info.token = SL_PP_QUESTION;
-         break;
+   case '^':
+      c = _pure_getc(context);
+      if (c == PURE_ERROR) {
+         return -1;
+      }
+      if (c == '^') {
+         out->token = SL_PP_XOR;
+      } else if (c == '=') {
+         out->token = SL_PP_BITXORASSIGN;
+      } else {
+         _pure_ungetc(context, c);
+         out->token = SL_PP_BITXOR;
+      }
+      break;
 
-      case ':':
-         input++;
-         info.token = SL_PP_COLON;
-         break;
+   case '|':
+      c = _pure_getc(context);
+      if (c == PURE_ERROR) {
+         return -1;
+      }
+      if (c == '|') {
+         out->token = SL_PP_OR;
+      } else if (c == '=') {
+         out->token = SL_PP_BITORASSIGN;
+      } else {
+         _pure_ungetc(context, c);
+         out->token = SL_PP_BITOR;
+      }
+      break;
 
-      case '\0':
-         info.token = SL_PP_EOF;
-         break;
+   case '?':
+      out->token = SL_PP_QUESTION;
+      break;
 
-      default:
-         if ((*input >= 'a' && *input <= 'z') ||
-             (*input >= 'A' && *input <= 'Z') ||
-             (*input == '_')) {
-            if (_tokenise_identifier(context, &input, &info)) {
-               free(out);
-               return -1;
-            }
-         } else if (*input >= '0' && *input <= '9') {
-            if (_tokenise_number(context, &input, &info)) {
-               free(out);
-               return -1;
-            }
-         } else {
-            info.data.other = *input++;
-            info.token = SL_PP_OTHER;
+   case ':':
+      out->token = SL_PP_COLON;
+      break;
+
+   case '\0':
+      out->token = SL_PP_EOF;
+      break;
+
+   case PURE_ERROR:
+      return -1;
+
+   default:
+      if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_') {
+         _pure_ungetc(context, c);
+         if (_tokenise_identifier(context, out)) {
+            return -1;
          }
+      } else if (c >= '0' && c <= '9') {
+         _pure_ungetc(context, c);
+         if (_tokenise_number(context, out)) {
+            return -1;
+         }
+      } else {
+         out->data.other = c;
+         out->token = SL_PP_OTHER;
+      }
+   }
+
+   return 0;
+}
+
+
+int
+sl_pp_tokenise(struct sl_pp_context *context,
+               const char *input,
+               const struct sl_pp_purify_options *options,
+               struct sl_pp_token_info **output)
+{
+   struct sl_pp_token_info *out = NULL;
+   unsigned int out_len = 0;
+   unsigned int out_max = 0;
+
+   sl_pp_purify_state_init(&context->pure, input, options);
+
+   for (;;) {
+      struct sl_pp_token_info info;
+
+      if (sl_pp_token_get(context, &info)) {
+         free(out);
+         return -1;
       }
 
       if (out_len >= out_max) {
diff --git a/src/glsl/pp/sl_pp_token.h b/src/glsl/pp/sl_pp_token.h
index 29c7571711e..7a8fa2f1b9d 100644
--- a/src/glsl/pp/sl_pp_token.h
+++ b/src/glsl/pp/sl_pp_token.h
@@ -123,6 +123,7 @@ struct sl_pp_token_info {
 int
 sl_pp_tokenise(struct sl_pp_context *context,
                const char *input,
+               const struct sl_pp_purify_options *options,
                struct sl_pp_token_info **output);
 
 #endif /* SL_PP_TOKEN_H */