diff options
| -rw-r--r-- | .gitlab-ci.yml | 19 | ||||
| -rw-r--r-- | .gitlab-ci/build-apitrace.sh | 18 | ||||
| -rw-r--r-- | .gitlab-ci/build-deqp-gl.sh | 2 | ||||
| -rw-r--r-- | .gitlab-ci/build-renderdoc.sh | 17 | ||||
| -rw-r--r-- | .gitlab-ci/container/x86_test-gl.sh | 33 | ||||
| -rwxr-xr-x | .gitlab-ci/prepare-artifacts.sh | 3 | ||||
| -rw-r--r-- | .gitlab-ci/traces.yml | 17 | ||||
| -rwxr-xr-x | .gitlab-ci/tracie-runner.sh | 33 | ||||
| -rw-r--r-- | .gitlab-ci/tracie/README.md | 126 | ||||
| -rw-r--r-- | .gitlab-ci/tracie/dump_trace_images.py | 134 | ||||
| -rw-r--r-- | .gitlab-ci/tracie/image_checksum.py | 39 | ||||
| -rw-r--r-- | .gitlab-ci/tracie/query_traces_yaml.py | 107 | ||||
| -rwxr-xr-x | .gitlab-ci/tracie/renderdoc_dump_images.py | 106 | ||||
| -rw-r--r-- | .gitlab-ci/tracie/tests/test-data/trace1/magenta.testtrace | 1 | ||||
| -rw-r--r-- | .gitlab-ci/tracie/tests/test-data/trace2/olive.testtrace | 1 | ||||
| -rwxr-xr-x | .gitlab-ci/tracie/tests/test.sh | 214 | ||||
| -rw-r--r-- | .gitlab-ci/tracie/tests/traces.yml | 9 | ||||
| -rw-r--r-- | .gitlab-ci/tracie/traceutil.py | 58 | ||||
| -rwxr-xr-x | .gitlab-ci/tracie/tracie.sh | 123 |
19 files changed, 1056 insertions, 4 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9e70ce88233..4b6c97a6b69 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -125,7 +125,7 @@ x86_build: x86_test-gl: extends: x86_build variables: - DEBIAN_TAG: &x86_test-gl "2020-01-30" + DEBIAN_TAG: &x86_test-gl "2020-02-14" # Debian 10 based x86 test image for VK x86_test-vk: @@ -705,3 +705,20 @@ radv_polaris10_vkcts: DEQP_SKIPS: deqp-radv-polaris10-skips.txt tags: - polaris10 + +.traces-test: + extends: + - .test-gl + cache: + key: ${CI_JOB_NAME} + paths: + - .git-lfs-storage/ + script: + - ./artifacts/tracie-runner.sh + +llvmpipe-traces: + variables: + LIBGL_ALWAYS_SOFTWARE: "true" + GALLIUM_DRIVER: "llvmpipe" + DEVICE_NAME: "vmware-llvmpipe" + extends: .traces-test diff --git a/.gitlab-ci/build-apitrace.sh b/.gitlab-ci/build-apitrace.sh new file mode 100644 index 00000000000..785a5ae52e7 --- /dev/null +++ b/.gitlab-ci/build-apitrace.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -ex + +APITRACE_VERSION="9.0" + +git clone https://github.com/apitrace/apitrace.git --single-branch --no-checkout /apitrace +pushd /apitrace +git checkout "$APITRACE_VERSION" +cmake -G Ninja -B_build -H. -DCMAKE_BUILD_TYPE=Release -DENABLE_GUI=False +ninja -C _build -j4 +mkdir build +cp _build/apitrace build +cp _build/glretrace build +cp _build/eglretrace build +strip build/* +find . -not -path './build' -not -path './build/*' -delete +popd diff --git a/.gitlab-ci/build-deqp-gl.sh b/.gitlab-ci/build-deqp-gl.sh index 13c684b31a6..57ce554f660 100644 --- a/.gitlab-ci/build-deqp-gl.sh +++ b/.gitlab-ci/build-deqp-gl.sh @@ -1,3 +1,5 @@ +#!/bin/bash + git config --global user.email "[email protected]" git config --global user.name "Mesa CI" git clone \ diff --git a/.gitlab-ci/build-renderdoc.sh b/.gitlab-ci/build-renderdoc.sh new file mode 100644 index 00000000000..31c8c014b8a --- /dev/null +++ b/.gitlab-ci/build-renderdoc.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -ex + +RENDERDOC_VERSION=6653316a62f6168b3e45040358cb77612dcffcb8 + +git clone https://github.com/baldurk/renderdoc.git --single-branch --no-checkout /renderdoc +pushd /renderdoc +git checkout "$RENDERDOC_VERSION" +cmake -G Ninja -B_build -H. -DENABLE_QRENDERDOC=false -DCMAKE_BUILD_TYPE=Release +ninja -C _build -j4 +mkdir -p build/lib +cp _build/lib/renderdoc.so build/lib +cp _build/lib/librenderdoc.so build/lib +strip build/lib/* +find . -not -path './build' -not -path './build/*' -delete +popd diff --git a/.gitlab-ci/container/x86_test-gl.sh b/.gitlab-ci/container/x86_test-gl.sh index 961ad2accc6..6a0a78183e0 100644 --- a/.gitlab-ci/container/x86_test-gl.sh +++ b/.gitlab-ci/container/x86_test-gl.sh @@ -28,33 +28,49 @@ EOF apt-get dist-upgrade -y apt-get install -y --no-remove \ + autoconf \ + automake \ cmake \ g++ \ git \ + git-lfs \ gcc \ libexpat1 \ libgbm-dev \ libgles2-mesa-dev \ + libpcre32-3 \ + libpcre3-dev \ libpng16-16 \ libpng-dev \ + libpython3.7 \ libvulkan1 \ libvulkan-dev \ libwaffle-dev \ libwayland-server0 \ + libxcb-keysyms1 \ + libxcb-keysyms1-dev \ libxcb-xfixes0 \ libxkbcommon0 \ libxkbcommon-dev \ libxrender1 \ libxrender-dev \ libllvm9 \ + make \ meson \ patch \ pkg-config \ + python \ + python3.7 \ + python3.7-dev \ python3-distutils \ python3-mako \ python3-numpy \ + python3-pil \ + python3-pilkit \ python3-six \ - python \ + python3-yaml \ + qt5-default \ + qt5-qmake \ waffle-utils \ xauth \ xvfb \ @@ -73,26 +89,37 @@ apt-get install -y --no-remove \ . .gitlab-ci/build-deqp-gl.sh +############### Build apitrace + +. .gitlab-ci/build-apitrace.sh + +############### Build renderdoc + +. .gitlab-ci/build-renderdoc.sh ############### Uninstall the build software apt-get purge -y \ + autoconf \ + automake \ cmake \ g++ \ gcc \ - git \ gnupg \ libc6-dev \ libgbm-dev \ libgles2-mesa-dev \ + libpcre3-dev \ libpng-dev \ libwaffle-dev \ + libxcb-keysyms1-dev \ libxkbcommon-dev \ libxrender-dev \ + make \ meson \ patch \ pkg-config \ - python \ + python3.7-dev \ python3-distutils apt-get autoremove -y --purge diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh index b1424b12102..f6a3f1beee4 100755 --- a/.gitlab-ci/prepare-artifacts.sh +++ b/.gitlab-ci/prepare-artifacts.sh @@ -26,6 +26,9 @@ mkdir -p artifacts/ cp VERSION artifacts/ cp -Rp .gitlab-ci/deqp* artifacts/ cp -Rp .gitlab-ci/piglit artifacts/ +cp -Rp .gitlab-ci/traces.yml artifacts/ +cp -Rp .gitlab-ci/tracie artifacts/ +cp -Rp .gitlab-ci/tracie-runner.sh artifacts/ # Tar up the install dir so that symlinks and hardlinks aren't each # packed separately in the zip file. diff --git a/.gitlab-ci/traces.yml b/.gitlab-ci/traces.yml new file mode 100644 index 00000000000..d390f185c20 --- /dev/null +++ b/.gitlab-ci/traces.yml @@ -0,0 +1,17 @@ +traces-db: + repo: "https://gitlab.freedesktop.org/gfx-ci/tracie/traces-db" + commit: "595235059fc84d7b03930aa0262ebca091d8260f" + +traces: + - path: glmark2/desktop-blur-radius=5:effect=blur:passes=1:separable=true:windows=4.rdc + expectations: + - device: vmware-llvmpipe + checksum: 8867f3a41f180626d0d4b7661ff5c0f4 + - path: glmark2/jellyfish.rdc + expectations: + - device: vmware-llvmpipe + checksum: e0fe979fee129c0ed42a3059d1a4e1c9 + - path: glxgears/glxgears.trace + expectations: + - device: vmware-llvmpipe + checksum: 02aca9b4b4ad6fd60331df6e4f87f2cd diff --git a/.gitlab-ci/tracie-runner.sh b/.gitlab-ci/tracie-runner.sh new file mode 100755 index 00000000000..7b5db08e105 --- /dev/null +++ b/.gitlab-ci/tracie-runner.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +set -ex + +ARTIFACTS="$(pwd)/artifacts" + +# Set up the driver environment. +export LD_LIBRARY_PATH="$(pwd)/install/lib/" + +# Set environment for renderdoc libraries. +export PYTHONPATH="$PYTHONPATH:/renderdoc/build/lib" +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/renderdoc/build/lib" + +# Perform a self-test to ensure tracie is working properly. +"$ARTIFACTS/tracie/tests/test.sh" + +ret=0 + +# The renderdoc version we use can handle surfaceless. +EGL_PLATFORM=surfaceless DISPLAY= \ + "$ARTIFACTS/tracie/tracie.sh" "$ARTIFACTS/traces.yml" renderdoc \ + || ret=1 + +# We need a newer waffle to use surfaceless with apitrace. For now run with +# xvfb. +xvfb-run --server-args="-noreset" sh -c \ + "set -ex; \ + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH; \ + export PATH=/apitrace/build:\$PATH; \ + \"$ARTIFACTS/tracie/tracie.sh\" \"$ARTIFACTS/traces.yml\" apitrace" \ + || ret=1 + +exit $ret diff --git a/.gitlab-ci/tracie/README.md b/.gitlab-ci/tracie/README.md new file mode 100644 index 00000000000..5c1f8a881ac --- /dev/null +++ b/.gitlab-ci/tracie/README.md @@ -0,0 +1,126 @@ +Tracie - Mesa Traces Continuous Integration System +================================================== + +Home of the Mesa trace testing effort. + +### Traces definition file + +The trace definition file contains information about the git repo/commit to get +the traces from, and a list of the traces to run along with their expected image +checksums on each device. An example: + +```yaml +traces-db: + repo: https://gitlab.freedesktop.org/gfx-ci/tracie/traces-db + commit: master + +traces: + - path: glmark2/jellyfish.rdc + expectations: + - device: intel-0x3185 + checksum: 58359ea4caf6ad44c6b65526881bbd17 + - device: vmware-llvmpipe + checksum: d82267c25a0decdad7b563c56bb81106 + - path: supertuxkart/supertuxkart-antediluvian-abyss.rdc + expectations: + - device: intel-0x3185 + checksum: ff827f7eb069afd87cc305a422cba939 +``` + +The traces-db entry can be absent, in which case it is assumed that the +current directory is the traces-db directory. + +Traces that don't have an expectation for the current device are skipped +during trace replay. + +Adding a new trace to the list involves commiting the trace to the git repo and +adding an entry to the `traces` list. The reference checksums can be calculated +with the [image_checksum.py](.gitlab-ci/tracie/image_checksum.py) script. +Alternatively, an arbitrary checksum can be used, and during replay (see below) +the scripts will report the mismatch and expected checksum. + +### Trace-db repos + +The trace-db repos are assumed to be git repositories using LFS for their trace +files. This is so that trace files can be potentially checked out and replayed +individually, thus reducing storage requirements during CI runs. + +### Enabling trace testing on a new device + +To enable trace testing on a new device: + +1. Create a new job in .gitlab-ci.yml. The job will need to be tagged + to run on runners with the appropriate hardware. Use the `.traces-test` + template job as a base, and make sure you set a unique value for the + `DEVICE_NAME` variable: + + ```yaml + my-hardware-traces: + variables: + DEVICE_NAME: "myhardware" + extends: .traces-test + ``` + +2. Update the .gitlab-ci/traces.yml file with expectations for the new device. + Ensure that the device name used in the expectations matches the one + set in the job. For more information, and tips about how to calculate + the checksums, see the section describing the trace definition files. + +### Trace files + +Tracie supports both renderdoc (.rdc) and apitrace (.trace) files. Trace files +need to have the correct extension so that tracie can detect them properly. + +The trace files that are contained in public traces-db repositories must be +legally redistributable. This is typically true for FOSS games and +applications. Traces for proprietary games and application are typically not +redistributable, unless specific redistribution rights have been granted by the +publisher. + +### Replaying traces + +Mesa traces CI uses a set of scripts to replay traces and check the output +against reference checksums. + +The high level script [tracie.sh](.gitlab-ci/tracie/tracie.sh) accepts +a traces definition file and the type of traces (apitrace/renderdoc) to run: + + tracie.sh .gitlab-ci/traces.yml renderdoc + +tracie.sh copies produced artifacts to the `$CI_PROJECT_DIR/result` +directory. By default, created images from traces are only stored in case of a +checksum mismatch. The `TRACIE_STORE_IMAGES` CI/environment variable can be set +to `1` to force storing images, e.g., to get a complete set of reference +images. + +The `tracie.sh` script requires that the environment variable `DEVICE_NAME` is +properly set for the target machine, and matches the `device` field of the +relevant trace expectations in the used `traces.yml` file. + +At a lower level the +[dump_trace_images.py](.gitlab-ci/tracie/dump_trace_images.py) script is +called, which replays a trace, dumping a set of images in the process. By +default only the image corresponding to the last frame of the trace is dumped, +but this can be changed with the `--calls` parameter. The dumped images are +stored in a subdirectory `test/<device-name>` next to the trace file itself, +with names of the form `tracefilename-callnum.png`. The full log of any +commands used while dumping the images is also saved in a file in the +'test/<device-name>' subdirectory, named after the trace name with '.log' +appended. + +Examples: + + python3 dump_traces_images.py --device-name=vmware-llvmpipe mytrace.trace + python3 dump_traces_images.py --device-name=vmware-llvmpipe --calls=2075,3300 mytrace.trace + +### Running the replay scripts locally + +It's often useful, especially during development, to be able to run the scripts +locally. The scripts require a recent version of apitrace being in the path, +and also the renderdoc python module being available. + +To ensure python3 can find the renderdoc python module you need to set +`PYTHONPATH` to point to the location of `renderdoc.so` (binary python modules) +and `LD_LIBRARY_PATH` to point to the location of `librenderdoc.so`. In the +renderdoc build tree, both of these are in `renderdoc/<builddir>/lib`. Note +that renderdoc doesn't install the `renderdoc.so` python module. diff --git a/.gitlab-ci/tracie/dump_trace_images.py b/.gitlab-ci/tracie/dump_trace_images.py new file mode 100644 index 00000000000..66a99f7efe2 --- /dev/null +++ b/.gitlab-ci/tracie/dump_trace_images.py @@ -0,0 +1,134 @@ +#!/usr/bin/python3 + +# Copyright (c) 2019 Collabora Ltd +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# SPDX-License-Identifier: MIT + +import argparse +import os +import sys +import subprocess +from pathlib import Path +from traceutil import trace_type_from_filename, TraceType + +def log(severity, msg, end='\n'): + print("[dump_trace_images] %s: %s" % (severity, msg), flush=True, end=end) + +def log_result(msg): + print(msg, flush=True) + +def run_logged_command(cmd, log_path): + ret = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + logoutput = ("[dump_trace_images] Running: %s\n" % " ".join(cmd)).encode() + \ + ret.stdout + log_path.parent.mkdir(parents=True, exist_ok=True) + with log_path.open(mode='wb') as log: + log.write(logoutput) + if ret.returncode: + raise RuntimeError( + logoutput.decode(errors='replace') + + "[dump_traces_images] Process failed with error code: %d" % ret.returncode) + +def get_last_apitrace_frame_call(trace_path): + cmd = ["apitrace", "dump", "--calls=frame", str(trace_path)] + ret = subprocess.run(cmd, stdout=subprocess.PIPE) + for l in reversed(ret.stdout.decode(errors='replace').splitlines()): + s = l.split(None, 1) + if len(s) >= 1 and s[0].isnumeric(): + return int(s[0]) + return -1 + +def dump_with_apitrace(trace_path, calls, device_name): + outputdir = str(trace_path.parent / "test" / device_name) + os.makedirs(outputdir, exist_ok=True) + outputprefix = str(Path(outputdir) / trace_path.name) + "-" + if len(calls) == 0: + calls = [str(get_last_apitrace_frame_call(trace_path))] + cmd = ["apitrace", "dump-images", "--calls=" + ','.join(calls), + "-o", outputprefix, str(trace_path)] + log_path = Path(outputdir) / (trace_path.name + ".log") + run_logged_command(cmd, log_path) + +def dump_with_renderdoc(trace_path, calls, device_name): + outputdir = str(trace_path.parent / "test" / device_name) + script_path = Path(os.path.dirname(os.path.abspath(__file__))) + cmd = [str(script_path / "renderdoc_dump_images.py"), str(trace_path), outputdir] + cmd.extend(calls) + log_path = Path(outputdir) / (trace_path.name + ".log") + run_logged_command(cmd, log_path) + +def dump_with_testtrace(trace_path, calls, device_name): + from PIL import Image + outputdir_path = trace_path.parent / "test" / device_name + outputdir_path.mkdir(parents=True, exist_ok=True) + with trace_path.open() as f: + rgba = f.read() + color = [int(rgba[0:2], 16), int(rgba[2:4], 16), + int(rgba[4:6], 16), int(rgba[6:8], 16)] + if len(calls) == 0: calls = ["0"] + for c in calls: + outputfile = str(outputdir_path / trace_path.name) + "-" + c + ".png" + log_path = outputdir_path / (trace_path.name + ".log") + with log_path.open(mode='w') as log: + log.write("Writing RGBA: %s to %s" % (rgba, outputfile)) + Image.frombytes('RGBA', (32, 32), bytes(color * 32 * 32)).save(outputfile) + +def dump_from_trace(trace_path, calls, device_name): + log("Info", "Dumping trace %s" % trace_path, end='... ') + trace_type = trace_type_from_filename(trace_path.name) + try: + if trace_type == TraceType.APITRACE: + dump_with_apitrace(trace_path, calls, device_name) + elif trace_type == TraceType.RENDERDOC: + dump_with_renderdoc(trace_path, calls, device_name) + elif trace_type == TraceType.TESTTRACE: + dump_with_testtrace(trace_path, calls, device_name) + else: + raise RuntimeError("Unknown tracefile extension") + log_result("OK") + return True + except Exception as e: + log_result("ERROR") + log("Debug", "=== Failure log start ===") + print(e) + log("Debug", "=== Failure log end ===") + return False + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('tracepath', help="trace to dump") + parser.add_argument('--device-name', required=True, + help="the name of the graphics device used to produce images") + parser.add_argument('--calls', required=False, + help="the call numbers from the trace to dump (default: last frame)") + + args = parser.parse_args() + if args.calls is not None: + args.calls = args.calls.split(",") + else: + args.calls = [] + + success = dump_from_trace(Path(args.tracepath), args.calls, args.device_name) + + sys.exit(0 if success else 1) + +if __name__ == "__main__": + main() diff --git a/.gitlab-ci/tracie/image_checksum.py b/.gitlab-ci/tracie/image_checksum.py new file mode 100644 index 00000000000..e920c1e7341 --- /dev/null +++ b/.gitlab-ci/tracie/image_checksum.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2019 Collabora Ltd +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# SPDX-License-Identifier: MIT + +import argparse +import hashlib +from PIL import Image + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('imagefile', help='image file to calculate checksum for') + + args = parser.parse_args() + + md5 = hashlib.md5(Image.open(args.imagefile).tobytes()) + print(md5.hexdigest()) + +if __name__ == "__main__": + main() diff --git a/.gitlab-ci/tracie/query_traces_yaml.py b/.gitlab-ci/tracie/query_traces_yaml.py new file mode 100644 index 00000000000..6ed069ba804 --- /dev/null +++ b/.gitlab-ci/tracie/query_traces_yaml.py @@ -0,0 +1,107 @@ +#!/usr/bin/python3 + +# Copyright (c) 2019 Collabora Ltd +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# SPDX-License-Identifier: MIT + +import argparse +import yaml +from traceutil import all_trace_type_names, trace_type_from_name +from traceutil import trace_type_from_filename + +def trace_devices(trace): + return [e['device'] for e in trace['expectations']] + +def cmd_traces_db_repo(args): + with open(args.file, 'r') as f: + y = yaml.safe_load(f) + print(y['traces-db']['repo']) + +def cmd_traces_db_commit(args): + with open(args.file, 'r') as f: + y = yaml.safe_load(f) + print(y['traces-db']['commit']) + +def cmd_traces(args): + with open(args.file, 'r') as f: + y = yaml.safe_load(f) + + traces = y['traces'] + traces = filter(lambda t: trace_type_from_filename(t['path']) in args.trace_types, + traces) + if args.device_name: + traces = filter(lambda t: args.device_name in trace_devices(t), traces) + + traces = list(traces) + + if len(traces) == 0: + return + + print('\n'.join((t['path'] for t in traces))) + +def cmd_checksum(args): + with open(args.file, 'r') as f: + y = yaml.safe_load(f) + + traces = y['traces'] + trace = next(t for t in traces if t['path'] == args.trace_path) + expectation = next(e for e in trace['expectations'] if e['device'] == args.device_name) + + print(expectation['checksum']) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--file', required=True, + help='the name of the yaml file') + + subparsers = parser.add_subparsers(help='sub-command help') + + parser_traces_db_repo = subparsers.add_parser('traces_db_repo') + parser_traces_db_repo.set_defaults(func=cmd_traces_db_repo) + + parser_traces_db_commit = subparsers.add_parser('traces_db_commit') + parser_traces_db_commit.set_defaults(func=cmd_traces_db_commit) + + parser_traces = subparsers.add_parser('traces') + parser_traces.add_argument('--device-name', required=False, + help="the name of the graphics device used to " + "produce images") + parser_traces.add_argument('--trace-types', required=False, + default=",".join(all_trace_type_names()), + help="the types of traces to look for in recursive " + "dir walks " "(by default all types)") + parser_traces.set_defaults(func=cmd_traces) + + parser_checksum = subparsers.add_parser('checksum') + parser_checksum.add_argument('--device-name', required=True, + help="the name of the graphics device used to " + "produce images") + parser_checksum.add_argument('trace_path') + parser_checksum.set_defaults(func=cmd_checksum) + + args = parser.parse_args() + if hasattr(args, 'trace_types'): + args.trace_types = [trace_type_from_name(t) for t in args.trace_types.split(",")] + + args.func(args) + +if __name__ == "__main__": + main() diff --git a/.gitlab-ci/tracie/renderdoc_dump_images.py b/.gitlab-ci/tracie/renderdoc_dump_images.py new file mode 100755 index 00000000000..f1252c11f25 --- /dev/null +++ b/.gitlab-ci/tracie/renderdoc_dump_images.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2019 Collabora Ltd +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# SPDX-License-Identifier: MIT + +import sys +from pathlib import Path + +import renderdoc as rd + +def findDrawWithEventId(controller, eventId): + for d in controller.GetDrawcalls(): + if d.eventId == eventId: + return d + + return None + +def dumpImage(controller, eventId, outputDir, tracefile): + draw = findDrawWithEventId(controller, eventId) + if draw is None: + raise RuntimeError("Couldn't find draw call with eventId " + str(eventId)) + + controller.SetFrameEvent(draw.eventId, True) + + texsave = rd.TextureSave() + + # Select the first color output + texsave.resourceId = draw.outputs[0] + + if texsave.resourceId == rd.ResourceId.Null(): + return + + filepath = Path(outputDir) + filepath.mkdir(parents = True, exist_ok = True) + filepath = filepath / (tracefile + "-" + str(int(draw.eventId)) + ".png") + + print("Saving image at eventId %d: %s to %s" % (draw.eventId, draw.name, filepath)) + + # Most formats can only display a single image per file, so we select the + # first mip and first slice + texsave.mip = 0 + texsave.slice.sliceIndex = 0 + + # For formats with an alpha channel, preserve it + texsave.alpha = rd.AlphaMapping.Preserve + texsave.destType = rd.FileType.PNG + controller.SaveTexture(texsave, str(filepath)) + +def loadCapture(filename): + cap = rd.OpenCaptureFile() + + status = cap.OpenFile(filename, '', None) + + if status != rd.ReplayStatus.Succeeded: + raise RuntimeError("Couldn't open file: " + str(status)) + if not cap.LocalReplaySupport(): + raise RuntimeError("Capture cannot be replayed") + + status,controller = cap.OpenCapture(rd.ReplayOptions(), None) + + if status != rd.ReplayStatus.Succeeded: + raise RuntimeError("Couldn't initialise replay: " + str(status)) + + return (cap, controller) + +def renderdoc_dump_images(filename, eventIds, outputDir): + rd.InitGlobalEnv(rd.GlobalEnvironment(), []) + cap,controller = loadCapture(filename); + + tracefile = Path(filename).name + + if len(eventIds) == 0: + eventIds.append(controller.GetDrawcalls()[-1].eventId) + + for eventId in eventIds: + dumpImage(controller, eventId, outputDir, tracefile) + + controller.Shutdown() + cap.Shutdown() + +if __name__ == "__main__": + if len(sys.argv) < 3: + raise RuntimeError("Usage: renderdoc_dump_images.py <trace> <outputdir> [<draw-id>...]") + + eventIds = [int(e) for e in sys.argv[3:]] + + renderdoc_dump_images(sys.argv[1], eventIds, sys.argv[2]) diff --git a/.gitlab-ci/tracie/tests/test-data/trace1/magenta.testtrace b/.gitlab-ci/tracie/tests/test-data/trace1/magenta.testtrace new file mode 100644 index 00000000000..2354cb56da6 --- /dev/null +++ b/.gitlab-ci/tracie/tests/test-data/trace1/magenta.testtrace @@ -0,0 +1 @@ +ff00ffff diff --git a/.gitlab-ci/tracie/tests/test-data/trace2/olive.testtrace b/.gitlab-ci/tracie/tests/test-data/trace2/olive.testtrace new file mode 100644 index 00000000000..825890f66c5 --- /dev/null +++ b/.gitlab-ci/tracie/tests/test-data/trace2/olive.testtrace @@ -0,0 +1 @@ +80800080 diff --git a/.gitlab-ci/tracie/tests/test.sh b/.gitlab-ci/tracie/tests/test.sh new file mode 100755 index 00000000000..7b4c9e822ce --- /dev/null +++ b/.gitlab-ci/tracie/tests/test.sh @@ -0,0 +1,214 @@ +#!/bin/sh + +TRACIE_DIR="$(dirname "$(readlink -f "$0")")/.." +TEST_DIR="" +TEST_EXIT=0 + +create_repo() { + repo="$(mktemp -d $TEST_DIR/repo.XXXXXXXXXX)" + cp -R "$TEST_DIR"/tests/test-data/* "$repo" + ( + cd "$repo"; + git init -q .; + git config user.email "[email protected]" + git config user.name "Me me" + git lfs track '*.testtrace' > /dev/null; + git add .; + git commit -q -a -m 'initial'; + ) + echo $repo +} + +destroy_repo() { + [ -d "$1"/.git ] && rm -rf "$1" +} + +assert() { + if ! $1; then + echo "Assertion failed: \"$1\"" + exit 1 + fi +} + +run_tracie() { + # Run tests for the .testtrace types, using the "test-device" device name. + DEVICE_NAME=test-device CI_PROJECT_DIR="$TEST_DIR" \ + "$TEST_DIR/tracie.sh" "$TEST_DIR/tests/traces.yml" testtrace +} + +cleanup() { + rm -rf "$TEST_DIR" +} + +prepare_for_run() { + TEST_DIR="$(mktemp -d -t tracie.test.XXXXXXXXXX)" + # Copy all the tracie scripts to the the test dir and later make that the + # CI_PROJECT_DIR for the run-tests.sh script. This avoids polluting the + # normal working dir with test result artifacts. + cp -R "$TRACIE_DIR"/. "$TEST_DIR" + trap cleanup EXIT + # Ensure we have a clean environment. + unset TRACIE_STORE_IMAGES +} + +run_test() { + prepare_for_run + log=$(mktemp) + if ($1 > "$log" 2>&1 ;); then + if [ -t 1 ]; then + echo "$1: \e[0;32mSuccess\e[0m" + else + echo "$1: Success" + fi + else + if [ -t 1 ]; then + echo "$1: \e[0;31mFail\e[0m" + else + echo "$1: Fail" + fi + cat "$log" + TEST_EXIT=1 + fi + rm "$log" + cleanup +} + +tracie_succeeds_if_all_images_match() { + repo="$(create_repo)" + cd "$repo" + + run_tracie + assert "[ $? = 0 ]" + + destroy_repo "$repo" +} + +tracie_fails_on_image_mismatch() { + repo="$(create_repo)" + cd "$repo" + + sed -i 's/5efda83854befe0155ff8517a58d5b51/8e0a801367e1714463475a824dab363b/g' \ + "$TEST_DIR/tests/traces.yml" + + run_tracie + assert "[ $? != 0 ]" + + destroy_repo "$repo" +} + +tracie_ignores_unspecified_trace_types() { + repo="$(create_repo)" + cd "$repo" + + echo " - path: trace1/empty.trace" >> "$TEST_DIR/tests/traces.yml" + echo " expectations:" >> "$TEST_DIR/tests/traces.yml" + echo " - device: test-device" >> "$TEST_DIR/tests/traces.yml" + echo " checksum: 000000000000000" >> "$TEST_DIR/tests/traces.yml" + # For the tests we only scan for the .testtrace type, + # so the .trace file added below should be ignored. + echo "empty" > trace1/empty.trace + git lfs track '*.trace' + git add trace1 + git commit -a -m 'break' + + run_tracie + assert "[ $? = 0 ]" + + destroy_repo "$repo" +} + +tracie_skips_traces_without_checksum() { + repo="$(create_repo)" + cd "$repo" + + echo " - path: trace1/red.testtrace" >> "$TEST_DIR/tests/traces.yml" + echo " expectations:" >> "$TEST_DIR/tests/traces.yml" + echo " - device: bla" >> "$TEST_DIR/tests/traces.yml" + echo " checksum: 000000000000000" >> "$TEST_DIR/tests/traces.yml" + # red.testtrace should be skipped, since it doesn't + # have any checksums for our device + echo "ff0000ff" > trace1/red.testtrace + git add trace1 + git commit -a -m 'red' + + run_tracie + assert "[ $? = 0 ]" + + destroy_repo "$repo" +} + +tracie_fails_on_dump_image_error() { + repo="$(create_repo)" + cd "$repo" + + # "invalid" should fail to parse as rgba and + # cause an error + echo "invalid" > trace1/magenta.testtrace + git add trace1 + git commit -a -m 'invalid' + + run_tracie + assert "[ $? != 0 ]" + + destroy_repo "$repo" +} + +tracie_stores_only_logs_on_checksum_match() { + repo="$(create_repo)" + cd "$repo" + + run_tracie + assert "[ $? = 0 ]" + + assert "[ -f "$TEST_DIR/results/trace1/test/test-device/magenta.testtrace.log" ]" + assert "[ -f "$TEST_DIR/results/trace2/test/test-device/olive.testtrace.log" ]" + + assert "[ ! -f "$TEST_DIR/results/trace1/test/test-device/magenta.testtrace-0.png" ]" + assert "[ ! -f "$TEST_DIR/results/trace2/test/test-device/olive.testtrace-0.png" ]" + + ls -lR "$TEST_DIR" + + destroy_repo "$repo" +} + +tracie_stores_images_on_checksum_mismatch() { + repo="$(create_repo)" + cd "$repo" + + sed -i 's/5efda83854befe0155ff8517a58d5b51/8e0a801367e1714463475a824dab363b/g' \ + "$TEST_DIR/tests/traces.yml" + + run_tracie + assert "[ $? != 0 ]" + + assert "[ ! -f "$TEST_DIR/results/trace1/test/test-device/magenta.testtrace-0.png" ]" + assert "[ -f "$TEST_DIR/results/trace2/test/test-device/olive.testtrace-0.png" ]" + + destroy_repo "$repo" +} + +tracie_stores_images_on_request() { + repo="$(create_repo)" + cd "$repo" + + (export TRACIE_STORE_IMAGES=1; run_tracie) + assert "[ $? = 0 ]" + + assert "[ -f "$TEST_DIR/results/trace1/test/test-device/magenta.testtrace-0.png" ]" + assert "[ -f "$TEST_DIR/results/trace2/test/test-device/olive.testtrace-0.png" ]" + + ls -lR "$TEST_DIR" + + destroy_repo "$repo" +} + +run_test tracie_succeeds_if_all_images_match +run_test tracie_fails_on_image_mismatch +run_test tracie_ignores_unspecified_trace_types +run_test tracie_skips_traces_without_checksum +run_test tracie_fails_on_dump_image_error +run_test tracie_stores_only_logs_on_checksum_match +run_test tracie_stores_images_on_checksum_mismatch +run_test tracie_stores_images_on_request + +exit $TEST_EXIT diff --git a/.gitlab-ci/tracie/tests/traces.yml b/.gitlab-ci/tracie/tests/traces.yml new file mode 100644 index 00000000000..dc6dd04b526 --- /dev/null +++ b/.gitlab-ci/tracie/tests/traces.yml @@ -0,0 +1,9 @@ +traces: + - path: trace1/magenta.testtrace + expectations: + - device: test-device + checksum: 8e0a801367e1714463475a824dab363b + - path: trace2/olive.testtrace + expectations: + - device: test-device + checksum: 5efda83854befe0155ff8517a58d5b51 diff --git a/.gitlab-ci/tracie/traceutil.py b/.gitlab-ci/tracie/traceutil.py new file mode 100644 index 00000000000..1b4de23ba5d --- /dev/null +++ b/.gitlab-ci/tracie/traceutil.py @@ -0,0 +1,58 @@ +# Copyright (c) 2019 Collabora Ltd +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# SPDX-License-Identifier: MIT + +import os +from pathlib import Path +from enum import Enum, auto + +class TraceType(Enum): + UNKNOWN = auto() + APITRACE = auto() + RENDERDOC = auto() + TESTTRACE = auto() + +_trace_type_info_map = { + TraceType.APITRACE : ("apitrace", ".trace"), + TraceType.RENDERDOC : ("renderdoc", ".rdc"), + TraceType.TESTTRACE : ("testtrace", ".testtrace") +} + +def all_trace_type_names(): + s = [] + for t,(name, ext) in _trace_type_info_map.items(): + if t != TraceType.UNKNOWN: + s.append(name) + return s + +def trace_type_from_name(tt_name): + for t,(name, ext) in _trace_type_info_map.items(): + if tt_name == name: + return t + + return TraceType.UNKNOWN + +def trace_type_from_filename(trace_file): + for t,(name, ext) in _trace_type_info_map.items(): + if trace_file.endswith(ext): + return t + + return TraceType.UNKNOWN diff --git a/.gitlab-ci/tracie/tracie.sh b/.gitlab-ci/tracie/tracie.sh new file mode 100755 index 00000000000..afae5be3365 --- /dev/null +++ b/.gitlab-ci/tracie/tracie.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash + +TRACIE_SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +TRACES_YAML="$(readlink -f "$1")" +TRACE_TYPE="$2" + +# Clone the traces-db repo without a checkout. Since we are dealing with +# git-lfs repositories, such clones are very lightweight. We check out +# individual files as needed at a later stage (see fetch_trace). +clone_traces_db_no_checkout() +{ + local repo="$1" + local commit="$2" + rm -rf traces-db + git clone --no-checkout -c lfs.storage="$CI_PROJECT_DIR/.git-lfs-storage" "$repo" traces-db + (cd traces-db; git reset "$commit" || git reset "origin/$commit") +} + +query_traces_yaml() +{ + python3 "$TRACIE_SCRIPT_DIR/query_traces_yaml.py" \ + --file "$TRACES_YAML" "$@" +} + +create_clean_git() +{ + rm -rf .clean_git + cp -R .git .clean_git +} + +restore_clean_git() +{ + rm -rf .git + cp -R .clean_git .git +} + +fetch_trace() +{ + local trace="${1//,/?}" + echo -n "[fetch_trace] Fetching $1... " + local output=$(git lfs pull -I "$trace" 2>&1) + local ret=0 + if [[ $? -ne 0 || ! -f "$1" ]]; then + echo "ERROR" + echo "$output" + ret=1 + else + echo "OK" + fi + # Restore a clean .git directory, effectively removing any downloaded + # git-lfs objects, in order to limit required storage. Note that the + # checked out trace file is still present at this point. We remove it + # when we are done with the trace replay at a later stage. + restore_clean_git + return $ret +} + +get_dumped_file() +{ + local trace="$1" + local tracedir="$(dirname "$trace")" + local tracename="$(basename "$trace")" + + find "$tracedir/test/$DEVICE_NAME" -name "$tracename*.$2" +} + +check_image() +{ + local trace="$1" + local image="$2" + + checksum=$(python3 "$TRACIE_SCRIPT_DIR/image_checksum.py" "$image") + expected=$(query_traces_yaml checksum --device-name "$DEVICE_NAME" "$trace") + if [[ "$checksum" = "$expected" ]]; then + echo "[check_image] Images match for $trace" + return 0 + else + echo "[check_image] Images differ for $trace (expected: $expected, actual: $checksum)" + echo "[check_image] For more information see https://gitlab.freedesktop.org/mesa/mesa/blob/master/.gitlab-ci/tracie/README.md" + return 1 + fi +} + +archive_artifact() +{ + mkdir -p "$CI_PROJECT_DIR/results" + cp --parents "$1" "$CI_PROJECT_DIR/results" +} + +if [[ -n "$(query_traces_yaml traces_db_repo)" ]]; then + clone_traces_db_no_checkout "$(query_traces_yaml traces_db_repo)" \ + "$(query_traces_yaml traces_db_commit)" + cd traces-db +else + echo "Warning: No traces-db entry in $TRACES_YAML, assuming traces-db is current directory" +fi + +# During git operations various git objects get created which +# may take up significant space. Store a clean .git instance, +# which we restore after various git operations to keep our +# storage consumption low. +create_clean_git + +ret=0 + +for trace in $(query_traces_yaml traces --device-name "$DEVICE_NAME" --trace-types "$TRACE_TYPE") +do + [[ -n "$(query_traces_yaml checksum --device-name "$DEVICE_NAME" "$trace")" ]] || + { echo "[fetch_trace] Skipping $trace since it has no checksums for $DEVICE_NAME"; continue; } + fetch_trace "$trace" || exit $? + python3 "$TRACIE_SCRIPT_DIR/dump_trace_images.py" --device-name "$DEVICE_NAME" "$trace" || exit $? + image="$(get_dumped_file "$trace" png)" + check_image "$trace" "$image" && check_succeeded=true || { ret=1; check_succeeded=false; } + if [[ "$check_succeeded" = false || "$TRACIE_STORE_IMAGES" = "1" ]]; then + archive_artifact "$image" + fi + archive_artifact "$(get_dumped_file "$trace" log)" + # Remove the downloaded trace file to reduce the total amount of storage + # that is required. + rm "$trace" +done + +exit $ret |