isl: Replace bash generator with python generator

This replaces the current bash generator with a python based generator
using mako. It's quite fast and works with both python 2.7 and python
3.5, and should work with 3.3+ and maybe even 3.2.

It produces an almost identical file except for a minor layout changes,
and the addition of a "generated file, do not edit" warning.

Cc: "12.0" <[email protected]>
Signed-off-by: Dylan Baker <[email protected]>
Reviewed-by: Jason Ekstrand <[email protected]>
Reviewed-by: Emil Velikov <[email protected]>

4 files changed, 217 insertions, 137 deletions

diff --git a/src/intel/isl/Android.mk b/src/intel/isl/Android.mk
index 31349813527..ff0c8c9f307 100644
--- a/src/intel/isl/Android.mk
+++ b/src/intel/isl/Android.mk
@@ -139,14 +139,14 @@ LOCAL_GENERATED_SOURCES += $(addprefix $(intermediates)/, $(ISL_GENERATED_FILES)
 define bash-gen
 	@mkdir -p $(dir $@)
 	@echo "Gen Bash: $(PRIVATE_MODULE) <= $(notdir $(@))"
-	$(hide) $(PRIVATE_SCRIPT) < $(PRIVATE_CSV) > $@
+	$(hide) $(PRIVATE_SCRIPT) --csv $(PRIVATE_CSV) --out $@
 endef
 
 isl_format_layout_deps := \
-	$(LOCAL_PATH)/isl_format_layout_gen.bash \
+	$(LOCAL_PATH)/gen_format_layout.py \
 	$(LOCAL_PATH)/isl_format_layout.csv
 
-$(intermediates)/isl_format_layout.c: PRIVATE_SCRIPT := bash -c $(LOCAL_PATH)/isl_format_layout_gen.bash
+$(intermediates)/isl_format_layout.c: PRIVATE_SCRIPT := $(MESA_PYTHON2) $(LOCAL_PATH)/gen_format_layout.py
 $(intermediates)/isl_format_layout.c: PRIVATE_CSV := $(LOCAL_PATH)/isl_format_layout.csv
 $(intermediates)/isl_format_layout.c: $(isl_format_layout_deps)
 	$(call bash-gen)
diff --git a/src/intel/isl/Makefile.am b/src/intel/isl/Makefile.am
index 74f863ab697..1fd668371d7 100644
--- a/src/intel/isl/Makefile.am
+++ b/src/intel/isl/Makefile.am
@@ -1,4 +1,4 @@
-# Copyright 2015 Intel Corporation
+# Copyright 2015-2016 Intel Corporation
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -66,10 +66,12 @@ libisl_gen9_la_CFLAGS = $(libisl_la_CFLAGS) -DGEN_VERSIONx10=90
 
 BUILT_SOURCES = $(ISL_GENERATED_FILES)
 
-isl_format_layout.c: isl_format_layout_gen.bash \
+PYTHON_GEN = $(AM_V_GEN)$(PYTHON2) $(PYTHON_FLAGS)
+
+isl_format_layout.c: gen_format_layout.py \
                      isl_format_layout.csv
-	$(AM_V_GEN)$(srcdir)/isl_format_layout_gen.bash \
-	    <$(srcdir)/isl_format_layout.csv >$@
+	$(PYTHON_GEN) $(srcdir)/gen_format_layout.py \
+	    --csv $(srcdir)/isl_format_layout.csv --out $@
 
 # ----------------------------------------------------------------------------
 #  Tests
@@ -87,6 +89,6 @@ tests_isl_surf_get_image_offset_test_LDADD = \
 # ----------------------------------------------------------------------------
 
 EXTRA_DIST = \
-	isl_format_layout_gen.bash \
+	gen_format_layout.py \
 	isl_format_layout.csv \
 	README
diff --git a/src/intel/isl/gen_format_layout.py b/src/intel/isl/gen_format_layout.py
new file mode 100644
index 00000000000..d7f39000b26
--- /dev/null
+++ b/src/intel/isl/gen_format_layout.py
@@ -0,0 +1,207 @@
+# encoding=utf-8
+# Copyright © 2016 Intel Corporation
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""Generates isl_format_layout.c."""
+
+from __future__ import absolute_import, division, print_function
+import argparse
+import csv
+import re
+import textwrap
+
+from mako import template
+
+# Load the template, ensure that __future__.division is imported, and set the
+# bytes encoding to be utf-8. This last bit is important to getting simple
+# consistent behavior for python 3 when we get there.
+TEMPLATE = template.Template(
+    text=textwrap.dedent("""\
+        /* This file is autogenerated by gen_format_layout.py. DO NOT EDIT! */
+
+        /*
+         * Copyright 2015 Intel Corporation
+         *
+         *  Permission is hereby granted, free of charge, to any person obtaining a
+         *  copy of this software and associated documentation files (the "Software"),
+         *  to deal in the Software without restriction, including without limitation
+         *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
+         *  and/or sell copies of the Software, and to permit persons to whom the
+         *  Software is furnished to do so, subject to the following conditions:
+         *
+         *  The above copyright notice and this permission notice (including the next
+         *  paragraph) shall be included in all copies or substantial portions of the
+         *  Software.
+         *
+         *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+         *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+         *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+         *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+         *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+         *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+         *  IN THE SOFTWARE.
+         */
+
+        #include "isl.h"
+
+        const struct isl_format_layout
+        isl_format_layouts[] = {
+        % for format in formats:
+          [ISL_FORMAT_${format.name}] = {
+            .format = ISL_FORMAT_${format.name},
+            .name = "ISL_FORMAT_${format.name}",
+            .bs = ${format.bs},
+            .bw = ${format.bw},
+            .bh = ${format.bh},
+            .bd = ${format.bd},
+            .channels = {
+            % for mask in ['r', 'g', 'b', 'a', 'l', 'i', 'p']:
+              <% channel = getattr(format, mask, None) %>\\
+              % if channel.type is not None:
+                .${mask} = { ISL_${channel.type}, ${channel.size} },
+              % else:
+                .${mask} = {},
+              % endif
+            % endfor
+            },
+            .colorspace = ISL_COLORSPACE_${format.colorspace},
+            .txc = ISL_TXC_${format.txc},
+          },
+
+        % endfor
+        };
+    """),
+    future_imports=['division'],
+    output_encoding='utf-8')
+
+
+class Channel(object):
+    """Class representing a Channel.
+
+    Converts the csv encoded data into the format that the template (and thus
+    the consuming C code) expects.
+
+    """
+    # If the csv file grew very large this class could be put behind a factory
+    # to increase efficiency. Right now though it's fast enough that It didn't
+    # seem worthwhile to add all of the boilerplate
+    _types = {
+        'x': 'void',
+        'r': 'raw',
+        'un': 'unorm',
+        'sn': 'snorm',
+        'uf': 'ufloat',
+        'sf': 'sfloat',
+        'ux': 'ufixed',
+        'sx': 'sfixed',
+        'ui': 'uint',
+        'si': 'sint',
+        'us': 'uscaled',
+        'ss': 'sscaled',
+    }
+    _splitter = re.compile(r'\s*(?P<type>[a-z]+)(?P<size>[0-9]+)')
+
+    def __init__(self, line):
+        # If the line is just whitespace then just set everything to None to
+        # save on the regex cost and let the template skip on None.
+        if line.isspace():
+            self.size = None
+            self.type = None
+        else:
+            grouped = self._splitter.match(line)
+            self.type = self._types[grouped.group('type')].upper()
+            self.size = grouped.group('size')
+
+
+class Format(object):
+    """Class taht contains all values needed by the template."""
+    def __init__(self, line):
+        # pylint: disable=invalid-name
+        self.name = line[0].strip()
+
+        # Future division makes this work in python 2.
+        self.bs = int(line[1]) // 8
+        self.bw = line[2].strip()
+        self.bh = line[3].strip()
+        self.bd = line[4].strip()
+        self.r = Channel(line[5])
+        self.g = Channel(line[6])
+        self.b = Channel(line[7])
+        self.a = Channel(line[8])
+        self.l = Channel(line[9])
+        self.i = Channel(line[10])
+        self.p = Channel(line[11])
+
+        # alpha doesn't have a colorspace of it's own.
+        self.colorspace = line[12].strip().upper()
+        if self.colorspace in ['', 'ALPHA']:
+            self.colorspace = 'NONE'
+
+        # This sets it to the line value, or if it's an empty string 'NONE'
+        self.txc = line[13].strip().upper() or 'NONE'
+
+
+def reader(csvfile):
+    """Wrapper around csv.reader that skips comments and blanks."""
+    # csv.reader actually reads the file one line at a time (it was designed to
+    # open excel generated sheets), so hold the file until all of the lines are
+    # read.
+    with open(csvfile, 'r') as f:
+        for line in csv.reader(f):
+            if line and not line[0].startswith('#'):
+                yield line
+
+
+def main():
+    """Main function."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--csv', action='store', help='The CSV file to parse.')
+    parser.add_argument(
+        '--out',
+        action='store',
+        help='The location to put the generated C file.')
+    args = parser.parse_args()
+
+    # This generator opens and writes the file itself, and it does so in bytes
+    # mode. This solves both python 2 vs 3 problems and solves the locale
+    # problem: Unicode can be rendered even if the shell calling this script
+    # doesn't.
+    with open(args.out, 'wb') as f:
+        try:
+            # This basically does lazy evaluation and initialization, which
+            # saves on memory and startup overhead.
+            f.write(TEMPLATE.render(
+                formats=(Format(l) for l in reader(args.csv))))
+        except Exception:
+            # In the even there's an error this imports some helpers from mako
+            # to print a useful stack trace and prints it, then exits with
+            # status 1, if python is run with debug; otherwise it just raises
+            # the exception
+            if __debug__:
+                import sys
+                from mako import exceptions
+                print(exceptions.text_error_template().render(),
+                      file=sys.stderr)
+                sys.exit(1)
+            raise
+
+
+if __name__ == '__main__':
+    main()
diff --git a/src/intel/isl/isl_format_layout_gen.bash b/src/intel/isl/isl_format_layout_gen.bash
deleted file mode 100755
index e20da5523cd..00000000000
--- a/src/intel/isl/isl_format_layout_gen.bash
+++ /dev/null
@@ -1,129 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright 2015 Intel Corporation
-#
-#  Permission is hereby granted, free of charge, to any person obtaining a
-#  copy of this software and associated documentation files (the "Software"),
-#  to deal in the Software without restriction, including without limitation
-#  the rights to use, copy, modify, merge, publish, distribute, sublicense,
-#  and/or sell copies of the Software, and to permit persons to whom the
-#  Software is furnished to do so, subject to the following conditions:
-#
-#  The above copyright notice and this permission notice (including the next
-#  paragraph) shall be included in all copies or substantial portions of the
-#  Software.
-#
-#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
-#  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-#  IN THE SOFTWARE.
-
-set -eu
-set -o pipefail
-
-cat <<'EOF'
-/*
- * Copyright 2015 Intel Corporation
- *
- *  Permission is hereby granted, free of charge, to any person obtaining a
- *  copy of this software and associated documentation files (the "Software"),
- *  to deal in the Software without restriction, including without limitation
- *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
- *  and/or sell copies of the Software, and to permit persons to whom the
- *  Software is furnished to do so, subject to the following conditions:
- *
- *  The above copyright notice and this permission notice (including the next
- *  paragraph) shall be included in all copies or substantial portions of the
- *  Software.
- *
- *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- *  IN THE SOFTWARE.
- */
-
-#include "isl.h"
-
-const struct isl_format_layout
-isl_format_layouts[] = {
-EOF
-
-sed -r '
-# Delete comment lines and empty lines
-/^[[:space:]]*#/d
-/^[[:space:]]*$/d
-
-# Delete spaces
-s/[[:space:]]//g
-
-# Translate formats
-s/^([A-Za-z0-9_]+),*/ISL_FORMAT_\1,/
-
-# Translate data type of channels
-s/\<x([0-9]+),/ISL_VOID@\1,/g
-s/\<r([0-9]+),/ISL_RAW@\1,/g
-s/\<un([0-9]+),/ISL_UNORM@\1,/g
-s/\<sn([0-9]+),/ISL_SNORM@\1,/g
-s/\<uf([0-9]+),/ISL_UFLOAT@\1,/g
-s/\<sf([0-9]+),/ISL_SFLOAT@\1,/g
-s/\<ux([0-9]+),/ISL_UFIXED@\1,/g
-s/\<sx([0-9]+),/ISL_SFIXED@\1,/g
-s/\<ui([0-9]+),/ISL_UINT@\1,/g
-s/\<si([0-9]+),/ISL_SINT@\1,/g
-s/\<us([0-9]+),/ISL_USCALED@\1,/g
-s/\<ss([0-9]+),/ISL_SSCALED@\1,/g
-
-# Translate colorspaces
-# Interpret alpha-only formats as having no colorspace.
-s/\<(linear|srgb|yuv)\>/ISL_COLORSPACE_\1/
-s/\<alpha\>//
-
-# Translate texture compression
-s/\<(dxt|fxt|rgtc|bptc|etc|astc)([0-9]*)\>/ISL_TXC_\1\2/
-' |
-tr 'a-z' 'A-Z' | # Convert to uppersace
-while IFS=, read -r format bpb bw bh bd \
-                    red green blue alpha \
-                    luminance intensity palette \
-                    colorspace txc
-do
-    : ${colorspace:=ISL_COLORSPACE_NONE}
-    : ${txc:=ISL_TXC_NONE}
-
-    cat <<EOF
-   [$format] = {
-      .format = $format,
-      .name = "$format",
-      .bs = $((bpb/8)),
-      .bw = $bw, .bh = $bh, .bd = $bd,
-      .channels = {
-          .r = { $red },
-          .g = { $green },
-          .b = { $blue },
-          .a = { $alpha },
-          .l = { $luminance },
-          .i = { $intensity },
-          .p = { $palette },
-      },
-      .colorspace = $colorspace,
-      .txc = $txc,
-   },
-
-EOF
-done |
-sed -r '
-# Collapse empty channels
-s/\{  \}/{}/
-
-# Split non-empty channels into two members: base type and bit size
-s/@/, /
-'
-
-# Terminate the table
-printf '};\n'