swr: [rasterizer codegen] Refactor codegen

Move common codegen functions into gen_common.py.

v2: change gen_knobs.py to find the template file internally, like
the rest of the gen scripts.

Reviewed-by: Bruce Cherniak <[email protected]>

6 files changed, 206 insertions, 147 deletions

diff --git a/src/gallium/drivers/swr/rasterizer/codegen/gen_archrast.py b/src/gallium/drivers/swr/rasterizer/codegen/gen_archrast.py
index cbaa1beddec..efe42bb8dbf 100644
--- a/src/gallium/drivers/swr/rasterizer/codegen/gen_archrast.py
+++ b/src/gallium/drivers/swr/rasterizer/codegen/gen_archrast.py
@@ -24,25 +24,7 @@ from __future__ import print_function
 import os
 import sys
 import re
-import argparse
-from mako.template import Template
-from mako.exceptions import RichTraceback
-
-def write_template_to_string(template_filename, **kwargs):
-    try:
-        template = Template(filename=template_filename)
-        # Split + Join fixes line-endings for whatever platform you are using
-        return '\n'.join(template.render(**kwargs).splitlines())
-    except:
-        traceback = RichTraceback()
-        for (filename, lineno, function, line) in traceback.traceback:
-            print('File %s, line %s, in %s' % (filename, lineno, function))
-            print(line, '\n')
-        print('%s: %s' % (str(traceback.error.__class__.__name__), traceback.error))
-
-def write_template_to_file(template_filename, output_filename, **kwargs):
-    with open(output_filename, 'w') as outfile:
-        print(write_template_to_string(template_filename, **kwargs), file=outfile)
+from gen_common import ArgumentParser, MakoTemplateWriter
 
 def parse_event_fields(lines, idx, event_dict):
     field_names = []
@@ -141,7 +123,7 @@ def parse_protos(filename):
 def main():
 
     # Parse args...
-    parser = argparse.ArgumentParser()
+    parser = ArgumentParser()
     parser.add_argument('--proto', '-p', help='Path to proto file', required=True)
     parser.add_argument('--output', '-o', help='Output filename (i.e. event.hpp)', required=True)
     parser.add_argument('--gen_event_hpp', help='Generate event header', action='store_true', default=False)
@@ -172,7 +154,7 @@ def main():
         template_file = os.sep.join([curdir, 'templates', 'gen_ar_event.hpp'])
         output_fullpath = os.sep.join([output_dir, output_filename])
 
-        write_template_to_file(template_file, output_fullpath,
+        MakoTemplateWriter.to_file(template_file, output_fullpath,
                 filename=output_filename,
                 protos=protos)
 
@@ -182,7 +164,7 @@ def main():
         template_file = os.sep.join([curdir, 'templates', 'gen_ar_event.cpp'])
         output_fullpath = os.sep.join([output_dir, output_filename])
 
-        write_template_to_file(template_file, output_fullpath,
+        MakoTemplateWriter.to_file(template_file, output_fullpath,
                 filename=output_filename,
                 protos=protos)
 
@@ -192,7 +174,7 @@ def main():
         template_file = os.sep.join([curdir, 'templates', 'gen_ar_eventhandler.hpp'])
         output_fullpath = os.sep.join([output_dir, output_filename])
 
-        write_template_to_file(template_file, output_fullpath,
+        MakoTemplateWriter.to_file(template_file, output_fullpath,
                 filename=output_filename,
                 event_header='gen_ar_event.hpp',
                 protos=protos)
@@ -203,7 +185,7 @@ def main():
         template_file = os.sep.join([curdir, 'templates', 'gen_ar_eventhandlerfile.hpp'])
         output_fullpath = os.sep.join([output_dir, output_filename])
 
-        write_template_to_file(template_file, output_fullpath,
+        MakoTemplateWriter.to_file(template_file, output_fullpath,
                 filename=output_filename,
                 event_header='gen_ar_eventhandler.hpp',
                 protos=protos)
diff --git a/src/gallium/drivers/swr/rasterizer/codegen/gen_backends.py b/src/gallium/drivers/swr/rasterizer/codegen/gen_backends.py
index f2a22729dc2..242ab7a73e9 100644
--- a/src/gallium/drivers/swr/rasterizer/codegen/gen_backends.py
+++ b/src/gallium/drivers/swr/rasterizer/codegen/gen_backends.py
@@ -24,36 +24,14 @@
 
 from __future__ import print_function
 import itertools
-import math
-import argparse
 import os
 import sys
-from mako.template import Template
-from mako.exceptions import RichTraceback
-
-def write_template_to_string(template_filename, **kwargs):
-    try:
-        template = Template(filename=os.path.abspath(template_filename))
-        # Split + Join fixes line-endings for whatever platform you are using
-        return '\n'.join(template.render(**kwargs).splitlines())
-    except:
-        traceback = RichTraceback()
-        for (filename, lineno, function, line) in traceback.traceback:
-            print("File %s, line %s, in %s" % (filename, lineno, function))
-            print(line, "\n")
-        print("%s: %s" % (str(traceback.error.__class__.__name__), traceback.error))
-
-def write_template_to_file(template_filename, output_filename, **kwargs):
-    output_dirname = os.path.dirname(output_filename)
-    if not os.path.exists(output_dirname):
-        os.makedirs(output_dirname)
-    with open(output_filename, "w") as outfile:
-        print(write_template_to_string(template_filename, **kwargs), file=outfile)
+from gen_common import ArgumentParser, MakoTemplateWriter
 
 
 def main(args=sys.argv[1:]):
     thisDir = os.path.dirname(os.path.realpath(__file__))
-    parser = argparse.ArgumentParser("Generate files and initialization functions for all permutuations of BackendPixelRate.")
+    parser = ArgumentParser("Generate files and initialization functions for all permutuations of BackendPixelRate.")
     parser.add_argument('--dim', help="gBackendPixelRateTable array dimensions", nargs='+', type=int, required=True)
     parser.add_argument('--outdir', help="output directory", nargs='?', type=str, default=thisDir)
     parser.add_argument('--split', help="how many lines of initialization per file [0=no split]", nargs='?', type=int, default='512')
@@ -100,7 +78,7 @@ def main(args=sys.argv[1:]):
         for fileNum in range(numFiles):
             filename = baseCppName % str(fileNum)
             #print('Generating', filename)
-            write_template_to_file(
+            MakoTemplateWriter.to_file(
                 templateCpp,
                 baseCppName % str(fileNum),
                 cmdline=sys.argv,
@@ -112,7 +90,7 @@ def main(args=sys.argv[1:]):
         templateCmake = os.path.join(thisDir, 'templates', 'gen_backend.cmake')
         cmakeFile = os.path.join(args.outdir, 'gen_backends.cmake')
         #print('Generating', cmakeFile)
-        write_template_to_file(
+        MakoTemplateWriter.to_file(
             templateCmake,
             cmakeFile,
             cmdline=sys.argv,
diff --git a/src/gallium/drivers/swr/rasterizer/codegen/gen_common.py b/src/gallium/drivers/swr/rasterizer/codegen/gen_common.py
new file mode 100644
index 00000000000..07b455a4e1d
--- /dev/null
+++ b/src/gallium/drivers/swr/rasterizer/codegen/gen_common.py
@@ -0,0 +1,162 @@
+# Copyright (C) 2014-2016 Intel Corporation.   All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice (including the next
+# paragraph) shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+# Python source
+from __future__ import print_function
+import os
+import sys
+import argparse
+from mako.template import Template
+from mako.exceptions import RichTraceback
+
+
+#==============================================================================
+class MakoTemplateWriter:
+    '''
+        MakoTemplateWriter - Class (namespace) for functions to generate strings
+        or files using the Mako template module.
+
+        See http://docs.makotemplates.org/en/latest/ for
+        mako documentation.
+   '''
+    
+    @staticmethod
+    def to_string(template_filename, **kwargs):
+        '''
+            Write template data to a string object and return the string
+        '''
+        from mako.template      import Template
+        from mako.exceptions    import RichTraceback
+
+        try:
+            template = Template(filename=template_filename)
+            # Split + Join fixes line-endings for whatever platform you are using
+            return '\n'.join(template.render(**kwargs).splitlines())
+        except:
+            traceback = RichTraceback()
+            for (filename, lineno, function, line) in traceback.traceback:
+                print('File %s, line %s, in %s' % (filename, lineno, function))
+                print(line, '\n')
+            print('%s: %s' % (str(traceback.error.__class__.__name__), traceback.error))
+
+    @staticmethod
+    def to_file(template_filename, output_filename, **kwargs):
+        '''
+            Write template data to a file
+        '''
+        with open(output_filename, 'w') as outfile:
+            print(MakoTemplateWriter.to_string(template_filename, **kwargs), file=outfile)
+
+
+#==============================================================================
+class ArgumentParser(argparse.ArgumentParser):
+    '''
+    Subclass of argparse.ArgumentParser
+
+    Allow parsing from command files that start with @
+    Example:
+      >bt run @myargs.txt
+    
+    Contents of myargs.txt:
+      -m <machine>
+      --target cdv_win7
+    
+    The below function allows multiple args to be placed on the same text-file line.
+    The default is one token per line, which is a little cumbersome.
+    
+    Also allow all characters after a '#' character to be ignored.
+    '''
+    
+    #==============================================================================
+    class _HelpFormatter(argparse.RawTextHelpFormatter):
+        ''' Better help formatter for argument parser '''
+
+        def _split_lines(self, text, width):
+            ''' optimized split lines algorighm, indents split lines '''
+            lines = text.splitlines()
+            out_lines = []
+            if len(lines):
+                out_lines.append(lines[0])
+                for line in lines[1:]:
+                    out_lines.append('  ' + line)
+            return out_lines
+
+    #==============================================================================
+    def __init__(self, *args, **kwargs):
+        ''' Constructor.  Compatible with argparse.ArgumentParser(),
+            but with some modifications for better usage and help display.
+        '''
+        super(ArgumentParser, self).__init__(
+                *args,
+                fromfile_prefix_chars='@',
+                formatter_class=ArgumentParser._HelpFormatter,
+                **kwargs)
+
+    #==========================================================================
+    def convert_arg_line_to_args(self, arg_line):
+        ''' convert one line of parsed file to arguments '''
+        arg_line = arg_line.split('#', 1)[0]
+        if sys.platform == 'win32':
+            arg_line = arg_line.replace('\\', '\\\\')
+        for arg in shlex.split(arg_line):
+            if not arg.strip():
+                continue
+            yield arg
+
+    #==========================================================================
+    def _read_args_from_files(self, arg_strings):
+        ''' read arguments from files '''
+        # expand arguments referencing files
+        new_arg_strings = []
+        for arg_string in arg_strings:
+
+            # for regular arguments, just add them back into the list
+            if arg_string[0] not in self.fromfile_prefix_chars:
+                new_arg_strings.append(arg_string)
+
+            # replace arguments referencing files with the file content
+            else:
+                filename = arg_string[1:]
+
+                # Search in sys.path
+                if not os.path.exists(filename):
+                    for path in sys.path:
+                        filename = os.path.join(path, arg_string[1:])
+                        if os.path.exists(filename):
+                            break
+
+                try:
+                    args_file = open(filename)
+                    try:
+                        arg_strings = []
+                        for arg_line in args_file.read().splitlines():
+                            for arg in self.convert_arg_line_to_args(arg_line):
+                                arg_strings.append(arg)
+                        arg_strings = self._read_args_from_files(arg_strings)
+                        new_arg_strings.extend(arg_strings)
+                    finally:
+                        args_file.close()
+                except IOError:
+                    err = sys.exc_info()[1]
+                    self.error(str(err))
+
+        # return the modified argument list
+        return new_arg_strings
diff --git a/src/gallium/drivers/swr/rasterizer/codegen/gen_knobs.py b/src/gallium/drivers/swr/rasterizer/codegen/gen_knobs.py
index e722b84a9f1..2c271c7f5c5 100644
--- a/src/gallium/drivers/swr/rasterizer/codegen/gen_knobs.py
+++ b/src/gallium/drivers/swr/rasterizer/codegen/gen_knobs.py
@@ -23,59 +23,41 @@
 from __future__ import print_function
 import os
 import sys
-import argparse
 import knob_defs
-from mako.template import Template
-from mako.exceptions import RichTraceback
-
-def write_template_to_string(template_filename, **kwargs):
-    try:
-        template = Template(filename=os.path.abspath(template_filename))
-        # Split + Join fixes line-endings for whatever platform you are using
-        return '\n'.join(template.render(**kwargs).splitlines())
-    except:
-        traceback = RichTraceback()
-        for (filename, lineno, function, line) in traceback.traceback:
-            print("File %s, line %s, in %s" % (filename, lineno, function))
-            print(line, "\n")
-        print("%s: %s" % (str(traceback.error.__class__.__name__), traceback.error))
-
-def write_template_to_file(template_filename, output_filename, **kwargs):
-    output_dirname = os.path.dirname(output_filename)
-    if not os.path.exists(output_dirname):
-        os.makedirs(output_dirname)
-    with open(output_filename, "w") as outfile:
-        print(write_template_to_string(template_filename, **kwargs), file=outfile)
+from gen_common import MakoTemplateWriter, ArgumentParser
 
 def main(args=sys.argv[1:]):
 
     # parse args
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--input", "-i", help="Path to gen_knobs.cpp template", required=True)
+    parser = ArgumentParser()
     parser.add_argument("--output", "-o", help="Path to output file", required=True)
     parser.add_argument("--gen_h", "-gen_h", help="Generate gen_knobs.h", action="store_true", default=False)
     parser.add_argument("--gen_cpp", "-gen_cpp", help="Generate gen_knobs.cpp", action="store_true", required=False)
 
     args = parser.parse_args()
 
-    if args.input:
-        if args.gen_h:
-            write_template_to_file(args.input,
-                                   args.output,
-                                   cmdline=sys.argv,
-                                   filename='gen_knobs',
-                                   knobs=knob_defs.KNOBS,
-                                   includes=['core/knobs_init.h', 'common/os.h', 'sstream', 'iomanip'],
-                                   gen_header=True)
+    cur_dir = os.path.dirname(os.path.abspath(__file__))
+    template_file = os.path.join(cur_dir, 'templates', 'gen_knobs.cpp')
+
+    if args.gen_h:
+        MakoTemplateWriter.to_file(
+            template_file,
+            args.output,
+            cmdline=sys.argv,
+            filename='gen_knobs',
+            knobs=knob_defs.KNOBS,
+            includes=['core/knobs_init.h', 'common/os.h', 'sstream', 'iomanip'],
+            gen_header=True)
 
-        if args.gen_cpp:
-            write_template_to_file(args.input,
-                                   args.output,
-                                   cmdline=sys.argv,
-                                   filename='gen_knobs',
-                                   knobs=knob_defs.KNOBS,
-                                   includes=['core/knobs_init.h', 'common/os.h', 'sstream', 'iomanip'],
-                                   gen_header=False)
+    if args.gen_cpp:
+        MakoTemplateWriter.to_file(
+            template_file,
+            args.output,
+            cmdline=sys.argv,
+            filename='gen_knobs',
+            knobs=knob_defs.KNOBS,
+            includes=['core/knobs_init.h', 'common/os.h', 'sstream', 'iomanip'],
+            gen_header=False)
 
     return 0
 
diff --git a/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_ir_macros.py b/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_ir_macros.py
index bd8e48e4218..dbf56471eec 100644
--- a/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_ir_macros.py
+++ b/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_ir_macros.py
@@ -21,31 +21,8 @@
 
 from __future__ import print_function
 import os, sys, re
-import argparse
-import json as JSON
-import operator
-from mako.template import Template
-from mako.exceptions import RichTraceback
-
-def write_template_to_string(template_filename, **kwargs):
-    try:
-        template = Template(filename=os.path.abspath(template_filename))
-        # Split + Join fixes line-endings for whatever platform you are using
-        return '\n'.join(template.render(**kwargs).splitlines())
-    except:
-        traceback = RichTraceback()
-        for (filename, lineno, function, line) in traceback.traceback:
-            print('File %s, line %s, in %s' % (filename, lineno, function))
-            print(line, '\n')
-        print('%s: %s' % (str(traceback.error.__class__.__name__), traceback.error))
-
-def write_template_to_file(template_filename, output_filename, **kwargs):
-    output_dirname = os.path.dirname(output_filename)
-    if not os.path.exists(output_dirname):
-        os.makedirs(output_dirname)
-    with open(output_filename, 'w') as outfile:
-        print(write_template_to_string(template_filename, **kwargs), file=outfile)
-
+from gen_common import MakoTemplateWriter, ArgumentParser
+from argparse import FileType
 
 inst_aliases = {
     'SHUFFLE_VECTOR': 'VSHUFFLE',
@@ -215,7 +192,7 @@ def generate_gen_h(functions, output_dir):
             'args'      : ', '.join(func['arg_names']),
         })
 
-    write_template_to_file(
+    MakoTemplateWriter.to_file(
         template,
         output_filename,
         cmdline=sys.argv,
@@ -242,7 +219,7 @@ def generate_x86_h(output_dir):
             'intrin'    : inst[1],
         })
 
-    write_template_to_file(
+    MakoTemplateWriter.to_file(
         template,
         output_filename,
         cmdline=sys.argv,
@@ -259,8 +236,8 @@ def generate_x86_h(output_dir):
 def main():
 
     # Parse args...
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--input', '-i', type=argparse.FileType('r'), help='Path to IRBuilder.h', required=False)
+    parser = ArgumentParser()
+    parser.add_argument('--input', '-i', type=FileType('r'), help='Path to IRBuilder.h', required=False)
     parser.add_argument('--output-dir', '-o', action='store', dest='output', help='Path to output directory', required=True)
     parser.add_argument('--gen_h', help='Generate builder_gen.h', action='store_true', default=False)
     parser.add_argument('--gen_x86_h', help='Generate x86 intrinsics. No input is needed.', action='store_true', default=False)
diff --git a/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_types.py b/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_types.py
index 72e67978d9f..1e9593a1af1 100644
--- a/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_types.py
+++ b/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_types.py
@@ -21,30 +21,8 @@
 
 from __future__ import print_function
 import os, sys, re
-import argparse
-import json as JSON
-import operator
-from mako.template import Template
-from mako.exceptions import RichTraceback
-
-def write_template_to_string(template_filename, **kwargs):
-    try:
-        template = Template(filename=os.path.abspath(template_filename))
-        # Split + Join fixes line-endings for whatever platform you are using
-        return '\n'.join(template.render(**kwargs).splitlines())
-    except:
-        traceback = RichTraceback()
-        for (filename, lineno, function, line) in traceback.traceback:
-            print('File %s, line %s, in %s' % (filename, lineno, function))
-            print(line, '\n')
-        print('%s: %s' % (str(traceback.error.__class__.__name__), traceback.error))
-
-def write_template_to_file(template_filename, output_filename, **kwargs):
-    output_dirname = os.path.dirname(output_filename)
-    if not os.path.exists(output_dirname):
-        os.makedirs(output_dirname)
-    with open(output_filename, 'w') as outfile:
-        print(write_template_to_string(template_filename, **kwargs), file=outfile)
+from gen_common import MakoTemplateWriter, ArgumentParser
+from argparse import FileType
 
 '''
 '''
@@ -300,7 +278,7 @@ def gen_llvm_types(input_file, output_file):
     cur_dir = os.path.dirname(os.path.abspath(__file__))
     template = os.path.join(cur_dir, 'templates', 'gen_llvm.hpp')
 
-    write_template_to_file(
+    MakoTemplateWriter.to_file(
         template,
         output_file,
         cmdline=sys.argv,
@@ -315,8 +293,8 @@ def gen_llvm_types(input_file, output_file):
 def main():
 
     # Parse args...
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--input', '-i', type=argparse.FileType('r'),
+    parser = ArgumentParser()
+    parser.add_argument('--input', '-i', type=FileType('r'),
             help='Path to input file containing structs', required=True)
     parser.add_argument('--output', '-o', action='store',
             help='Path to output file', required=True)