diff options
Diffstat (limited to 'src/glsl/pp/sl_pp_token.c')
| -rw-r--r-- | src/glsl/pp/sl_pp_token.c | 859 |
1 files changed, 859 insertions, 0 deletions
diff --git a/src/glsl/pp/sl_pp_token.c b/src/glsl/pp/sl_pp_token.c new file mode 100644 index 00000000000..e9a60b6c508 --- /dev/null +++ b/src/glsl/pp/sl_pp_token.c @@ -0,0 +1,859 @@ +/************************************************************************** + * + * Copyright 2009 VMware, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. + * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR + * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ + +#include <assert.h> +#include <stdlib.h> +#include <string.h> +#include "sl_pp_public.h" +#include "sl_pp_context.h" +#include "sl_pp_token.h" + + +#define PURE_ERROR 256 + +static int +_pure_getc(struct sl_pp_context *context) +{ + char c; + unsigned int current_line; + + if (context->getc_buf_size) { + return context->getc_buf[--context->getc_buf_size]; + } + + if (sl_pp_purify_getc(&context->pure, &c, ¤t_line, context->error_msg, sizeof(context->error_msg)) == 0) { + return PURE_ERROR; + } + return c; +} + + +static void +_pure_ungetc(struct sl_pp_context *context, + int c) +{ + assert(c != PURE_ERROR); + + if (context->getc_buf_size == context->getc_buf_capacity) { + context->getc_buf_capacity += 64; + context->getc_buf = realloc(context->getc_buf, context->getc_buf_capacity * sizeof(char)); + assert(context->getc_buf); + } + + context->getc_buf[context->getc_buf_size++] = (char)c; +} + + +struct lookahead_state { + char buf[256]; + unsigned int pos; + struct sl_pp_context *context; +}; + + +static void +_lookahead_init(struct lookahead_state *lookahead, + struct sl_pp_context *context) +{ + lookahead->pos = 0; + lookahead->context = context; +} + + +static unsigned int +_lookahead_tell(const struct lookahead_state *lookahead) +{ + return lookahead->pos; +} + + +static const void * +_lookahead_buf(const struct lookahead_state *lookahead) +{ + return lookahead->buf; +} + + +static void +_lookahead_revert(struct lookahead_state *lookahead, + unsigned int pos) +{ + assert(pos <= lookahead->pos); + + while (lookahead->pos > pos) { + _pure_ungetc(lookahead->context, lookahead->buf[--lookahead->pos]); + } +} + + +static int +_lookahead_getc(struct lookahead_state *lookahead) +{ + int c; + + assert(lookahead->pos < sizeof(lookahead->buf) / sizeof(lookahead->buf[0])); + + c = _pure_getc(lookahead->context); + if (c != PURE_ERROR) { + lookahead->buf[lookahead->pos++] = (char)c; + } + return c; +} + + +static int +_is_identifier_char(char c) +{ + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; +} + + +static int +_tokenise_identifier(struct sl_pp_context *context, + struct sl_pp_token_info *out) +{ + int c; + char identifier[256]; /* XXX: Remove this artifical limit. */ + unsigned int i = 0; + + out->token = SL_PP_IDENTIFIER; + out->data.identifier = -1; + + c = _pure_getc(context); + if (c == PURE_ERROR) { + return -1; + } + identifier[i++] = (char)c; + for (;;) { + c = _pure_getc(context); + if (c == PURE_ERROR) { + return -1; + } + + if (_is_identifier_char((char)c)) { + if (i >= sizeof(identifier) / sizeof(char) - 1) { + strcpy(context->error_msg, "out of memory"); + _pure_ungetc(context, c); + while (i) { + _pure_ungetc(context, identifier[--i]); + } + return -1; + } + identifier[i++] = (char)c; + } else { + _pure_ungetc(context, c); + break; + } + } + identifier[i] = '\0'; + + out->data.identifier = sl_pp_context_add_unique_str(context, identifier); + if (out->data.identifier == -1) { + while (i) { + _pure_ungetc(context, identifier[--i]); + } + return -1; + } + + return 0; +} + + +/* + * Return the number of consecutive decimal digits in the input stream. + */ +static unsigned int +_parse_float_digits(struct lookahead_state *lookahead) +{ + unsigned int eaten; + + for (eaten = 0;; eaten++) { + unsigned int pos = _lookahead_tell(lookahead); + char c = _lookahead_getc(lookahead); + + if (c < '0' || c > '9') { + _lookahead_revert(lookahead, pos); + break; + } + } + return eaten; +} + + +/* + * Try to match one of the following patterns for the fractional part + * of a floating point number. + * + * digits . [digits] + * . digits + * + * Return 0 if the pattern could not be matched, otherwise the number + * of eaten characters from the input stream. + */ +static unsigned int +_parse_float_frac(struct lookahead_state *lookahead) +{ + unsigned int pos; + int c; + unsigned int eaten; + + pos = _lookahead_tell(lookahead); + c = _lookahead_getc(lookahead); + if (c == '.') { + eaten = _parse_float_digits(lookahead); + if (eaten) { + return eaten + 1; + } + _lookahead_revert(lookahead, pos); + return 0; + } + + _lookahead_revert(lookahead, pos); + eaten = _parse_float_digits(lookahead); + if (eaten) { + c = _lookahead_getc(lookahead); + if (c == '.') { + return eaten + 1 + _parse_float_digits(lookahead); + } + } + + _lookahead_revert(lookahead, pos); + return 0; +} + + +/* + * Try to match the following pattern for the exponential part + * of a floating point number. + * + * (e|E) [(+|-)] digits + * + * Return 0 if the pattern could not be matched, otherwise the number + * of eaten characters from the input stream. + */ +static unsigned int +_parse_float_exp(struct lookahead_state *lookahead) +{ + unsigned int pos, pos2; + int c; + unsigned int eaten, digits; + + pos = _lookahead_tell(lookahead); + c = _lookahead_getc(lookahead); + if (c != 'e' && c != 'E') { + _lookahead_revert(lookahead, pos); + return 0; + } + + pos2 = _lookahead_tell(lookahead); + c = _lookahead_getc(lookahead); + if (c == '-' || c == '+') { + eaten = 2; + } else { + _lookahead_revert(lookahead, pos2); + eaten = 1; + } + + digits = _parse_float_digits(lookahead); + if (!digits) { + _lookahead_revert(lookahead, pos); + return 0; + } + + return eaten + digits; +} + + +/* + * Try to match one of the following patterns for a floating point number. + * + * fract [exp] [(f|F)] + * digits exp [(f|F)] + * + * Return 0 if the pattern could not be matched, otherwise the number + * of eaten characters from the input stream. + */ +static unsigned int +_parse_float(struct lookahead_state *lookahead) +{ + unsigned int eaten; + + eaten = _parse_float_frac(lookahead); + if (eaten) { + unsigned int pos; + int c; + + eaten += _parse_float_exp(lookahead); + + pos = _lookahead_tell(lookahead); + c = _lookahead_getc(lookahead); + if (c == 'f' || c == 'F') { + eaten++; + } else { + _lookahead_revert(lookahead, pos); + } + + return eaten; + } + + eaten = _parse_float_digits(lookahead); + if (eaten) { + unsigned int exponent; + + exponent = _parse_float_exp(lookahead); + if (exponent) { + unsigned int pos; + int c; + + eaten += exponent; + + pos = _lookahead_tell(lookahead); + c = _lookahead_getc(lookahead); + if (c == 'f' || c == 'F') { + eaten++; + } else { + _lookahead_revert(lookahead, pos); + } + + return eaten; + } + } + + _lookahead_revert(lookahead, 0); + return 0; +} + + +static unsigned int +_parse_hex(struct lookahead_state *lookahead) +{ + int c; + unsigned int n; + + c = _lookahead_getc(lookahead); + if (c != '0') { + _lookahead_revert(lookahead, 0); + return 0; + } + + c = _lookahead_getc(lookahead); + if (c != 'x' && c != 'X') { + _lookahead_revert(lookahead, 0); + return 0; + } + + for (n = 2;;) { + unsigned int pos = _lookahead_tell(lookahead); + + c = _lookahead_getc(lookahead); + if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { + n++; + } else { + _lookahead_revert(lookahead, pos); + break; + } + } + + if (n > 2) { + return n; + } + + _lookahead_revert(lookahead, 0); + return 0; +} + + +static unsigned int +_parse_oct(struct lookahead_state *lookahead) +{ + int c; + unsigned int n; + + c = _lookahead_getc(lookahead); + if (c != '0') { + _lookahead_revert(lookahead, 0); + return 0; + } + + for (n = 1;;) { + unsigned int pos = _lookahead_tell(lookahead); + + c = _lookahead_getc(lookahead); + if ((c >= '0' && c <= '7')) { + n++; + } else { + _lookahead_revert(lookahead, pos); + break; + } + } + + return n; +} + + +static unsigned int +_parse_dec(struct lookahead_state *lookahead) +{ + unsigned int n = 0; + + for (;;) { + unsigned int pos = _lookahead_tell(lookahead); + int c = _lookahead_getc(lookahead); + + if ((c >= '0' && c <= '9')) { + n++; + } else { + _lookahead_revert(lookahead, pos); + break; + } + } + + return n; +} + + +static int +_tokenise_number(struct sl_pp_context *context, + struct sl_pp_token_info *out) +{ + struct lookahead_state lookahead; + unsigned int eaten; + unsigned int is_float = 0; + unsigned int pos; + int c; + char number[256]; /* XXX: Remove this artifical limit. */ + + _lookahead_init(&lookahead, context); + + eaten = _parse_float(&lookahead); + if (!eaten) { + eaten = _parse_hex(&lookahead); + if (!eaten) { + eaten = _parse_oct(&lookahead); + if (!eaten) { + eaten = _parse_dec(&lookahead); + } + } + } else { + is_float = 1; + } + + if (!eaten) { + strcpy(context->error_msg, "expected a number"); + return -1; + } + + pos = _lookahead_tell(&lookahead); + c = _lookahead_getc(&lookahead); + _lookahead_revert(&lookahead, pos); + + if (_is_identifier_char(c)) { + strcpy(context->error_msg, "expected a number"); + _lookahead_revert(&lookahead, 0); + return -1; + } + + if (eaten > sizeof(number) - 1) { + strcpy(context->error_msg, "out of memory"); + _lookahead_revert(&lookahead, 0); + return -1; + } + + assert(_lookahead_tell(&lookahead) == eaten); + + memcpy(number, _lookahead_buf(&lookahead), eaten); + number[eaten] = '\0'; + + if (is_float) { + out->token = SL_PP_FLOAT; + out->data._float = sl_pp_context_add_unique_str(context, number); + if (out->data._float == -1) { + _lookahead_revert(&lookahead, 0); + return -1; + } + } else { + out->token = SL_PP_UINT; + out->data._uint = sl_pp_context_add_unique_str(context, number); + if (out->data._uint == -1) { + _lookahead_revert(&lookahead, 0); + return -1; + } + } + + return 0; +} + + +static int +sl_pp_token_get(struct sl_pp_context *context, + struct sl_pp_token_info *out) +{ + int c = _pure_getc(context); + + switch (c) { + case ' ': + case '\t': + out->token = SL_PP_WHITESPACE; + break; + + case '\n': + out->token = SL_PP_NEWLINE; + break; + + case '#': + out->token = SL_PP_HASH; + break; + + case ',': + out->token = SL_PP_COMMA; + break; + + case ';': + out->token = SL_PP_SEMICOLON; + break; + + case '{': + out->token = SL_PP_LBRACE; + break; + + case '}': + out->token = SL_PP_RBRACE; + break; + + case '(': + out->token = SL_PP_LPAREN; + break; + + case ')': + out->token = SL_PP_RPAREN; + break; + + case '[': + out->token = SL_PP_LBRACKET; + break; + + case ']': + out->token = SL_PP_RBRACKET; + break; + + case '.': + { + int c2 = _pure_getc(context); + + if (c2 == PURE_ERROR) { + return -1; + } + if (c2 >= '0' && c2 <= '9') { + _pure_ungetc(context, c2); + _pure_ungetc(context, c); + if (_tokenise_number(context, out)) { + return -1; + } + } else { + _pure_ungetc(context, c2); + out->token = SL_PP_DOT; + } + } + break; + + case '+': + c = _pure_getc(context); + if (c == PURE_ERROR) { + return -1; + } + if (c == '+') { + out->token = SL_PP_INCREMENT; + } else if (c == '=') { + out->token = SL_PP_ADDASSIGN; + } else { + _pure_ungetc(context, c); + out->token = SL_PP_PLUS; + } + break; + + case '-': + c = _pure_getc(context); + if (c == PURE_ERROR) { + return -1; + } + if (c == '-') { + out->token = SL_PP_DECREMENT; + } else if (c == '=') { + out->token = SL_PP_SUBASSIGN; + } else { + _pure_ungetc(context, c); + out->token = SL_PP_MINUS; + } + break; + + case '~': + out->token = SL_PP_BITNOT; + break; + + case '!': + c = _pure_getc(context); + if (c == PURE_ERROR) { + return -1; + } + if (c == '=') { + out->token = SL_PP_NOTEQUAL; + } else { + _pure_ungetc(context, c); + out->token = SL_PP_NOT; + } + break; + + case '*': + c = _pure_getc(context); + if (c == PURE_ERROR) { + return -1; + } + if (c == '=') { + out->token = SL_PP_MULASSIGN; + } else { + _pure_ungetc(context, c); + out->token = SL_PP_STAR; + } + break; + + case '/': + c = _pure_getc(context); + if (c == PURE_ERROR) { + return -1; + } + if (c == '=') { + out->token = SL_PP_DIVASSIGN; + } else { + _pure_ungetc(context, c); + out->token = SL_PP_SLASH; + } + break; + + case '%': + c = _pure_getc(context); + if (c == PURE_ERROR) { + return -1; + } + if (c == '=') { + out->token = SL_PP_MODASSIGN; + } else { + _pure_ungetc(context, c); + out->token = SL_PP_MODULO; + } + break; + + case '<': + c = _pure_getc(context); + if (c == PURE_ERROR) { + return -1; + } + if (c == '<') { + c = _pure_getc(context); + if (c == PURE_ERROR) { + return -1; + } + if (c == '=') { + out->token = SL_PP_LSHIFTASSIGN; + } else { + _pure_ungetc(context, c); + out->token = SL_PP_LSHIFT; + } + } else if (c == '=') { + out->token = SL_PP_LESSEQUAL; + } else { + _pure_ungetc(context, c); + out->token = SL_PP_LESS; + } + break; + + case '>': + c = _pure_getc(context); + if (c == PURE_ERROR) { + return -1; + } + if (c == '>') { + c = _pure_getc(context); + if (c == PURE_ERROR) { + return -1; + } + if (c == '=') { + out->token = SL_PP_RSHIFTASSIGN; + } else { + _pure_ungetc(context, c); + out->token = SL_PP_RSHIFT; + } + } else if (c == '=') { + out->token = SL_PP_GREATEREQUAL; + } else { + _pure_ungetc(context, c); + out->token = SL_PP_GREATER; + } + break; + + case '=': + c = _pure_getc(context); + if (c == PURE_ERROR) { + return -1; + } + if (c == '=') { + out->token = SL_PP_EQUAL; + } else { + _pure_ungetc(context, c); + out->token = SL_PP_ASSIGN; + } + break; + + case '&': + c = _pure_getc(context); + if (c == PURE_ERROR) { + return -1; + } + if (c == '&') { + out->token = SL_PP_AND; + } else if (c == '=') { + out->token = SL_PP_BITANDASSIGN; + } else { + _pure_ungetc(context, c); + out->token = SL_PP_BITAND; + } + break; + + case '^': + c = _pure_getc(context); + if (c == PURE_ERROR) { + return -1; + } + if (c == '^') { + out->token = SL_PP_XOR; + } else if (c == '=') { + out->token = SL_PP_BITXORASSIGN; + } else { + _pure_ungetc(context, c); + out->token = SL_PP_BITXOR; + } + break; + + case '|': + c = _pure_getc(context); + if (c == PURE_ERROR) { + return -1; + } + if (c == '|') { + out->token = SL_PP_OR; + } else if (c == '=') { + out->token = SL_PP_BITORASSIGN; + } else { + _pure_ungetc(context, c); + out->token = SL_PP_BITOR; + } + break; + + case '?': + out->token = SL_PP_QUESTION; + break; + + case ':': + out->token = SL_PP_COLON; + break; + + case '\0': + out->token = SL_PP_EOF; + break; + + case PURE_ERROR: + return -1; + + default: + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_') { + _pure_ungetc(context, c); + if (_tokenise_identifier(context, out)) { + return -1; + } + } else if (c >= '0' && c <= '9') { + _pure_ungetc(context, c); + if (_tokenise_number(context, out)) { + return -1; + } + } else { + out->data.other = c; + out->token = SL_PP_OTHER; + } + } + + return 0; +} + + +int +sl_pp_tokenise(struct sl_pp_context *context, + const char *input, + const struct sl_pp_purify_options *options, + struct sl_pp_token_info **output) +{ + struct sl_pp_token_info *out = NULL; + unsigned int out_len = 0; + unsigned int out_max = 0; + + sl_pp_purify_state_init(&context->pure, input, options); + + for (;;) { + struct sl_pp_token_info info; + + if (sl_pp_token_get(context, &info)) { + free(out); + return -1; + } + + if (out_len >= out_max) { + unsigned int new_max = out_max; + + if (new_max < 0x100) { + new_max = 0x100; + } else if (new_max < 0x10000) { + new_max *= 2; + } else { + new_max += 0x10000; + } + + out = realloc(out, new_max * sizeof(struct sl_pp_token_info)); + if (!out) { + strcpy(context->error_msg, "out of memory"); + return -1; + } + out_max = new_max; + } + + out[out_len++] = info; + + if (info.token == SL_PP_EOF) { + break; + } + } + + *output = out; + return 0; +} |