diff options
Diffstat (limited to '.gitlab-ci/tracie')
| -rwxr-xr-x | .gitlab-ci/tracie/renderdoc_dump_images.py | 20 | ||||
| -rwxr-xr-x | .gitlab-ci/tracie/tests/test.sh | 100 | ||||
| -rw-r--r-- | .gitlab-ci/tracie/tracie.py | 167 | ||||
| -rwxr-xr-x | .gitlab-ci/tracie/tracie.sh | 123 |
4 files changed, 196 insertions, 214 deletions
diff --git a/.gitlab-ci/tracie/renderdoc_dump_images.py b/.gitlab-ci/tracie/renderdoc_dump_images.py index 689ee2d0a87..93e24b9ca25 100755 --- a/.gitlab-ci/tracie/renderdoc_dump_images.py +++ b/.gitlab-ci/tracie/renderdoc_dump_images.py @@ -22,9 +22,22 @@ # # SPDX-License-Identifier: MIT +import atexit +import os +import shutil import sys +import tempfile from pathlib import Path +def cleanup(dirpath): + shutil.rmtree(dirpath) + +dirpath = tempfile.mkdtemp() +atexit.register(cleanup, dirpath) +RENDERDOC_DEBUG_FILE = dirpath + "/renderdoc.log" + +# Needs to be in the environment before importing the module +os.environ['RENDERDOC_DEBUG_LOG_FILE'] = RENDERDOC_DEBUG_FILE import renderdoc as rd def findDrawWithEventId(controller, eventId): @@ -75,11 +88,16 @@ def loadCapture(filename): if not cap.LocalReplaySupport(): raise RuntimeError("Capture cannot be replayed") - status,controller = cap.OpenCapture(rd.ReplayOptions(), None) + status, controller = cap.OpenCapture(rd.ReplayOptions(), None) if status != rd.ReplayStatus.Succeeded: + if os.path.exists(RENDERDOC_DEBUG_FILE): + print(open(RENDERDOC_DEBUG_FILE, "r").read()) raise RuntimeError("Couldn't initialise replay: " + str(status)) + if os.path.exists(RENDERDOC_DEBUG_FILE): + open(RENDERDOC_DEBUG_FILE, "w").write("") + return (cap, controller) def renderdoc_dump_images(filename, eventIds, outputDir): diff --git a/.gitlab-ci/tracie/tests/test.sh b/.gitlab-ci/tracie/tests/test.sh index 4d52578c47e..cd0f08e4efd 100755 --- a/.gitlab-ci/tracie/tests/test.sh +++ b/.gitlab-ci/tracie/tests/test.sh @@ -4,25 +4,6 @@ TRACIE_DIR="$(dirname "$(readlink -f "$0")")/.." TEST_DIR="" TEST_EXIT=0 -create_repo() { - repo="$(mktemp -d $TEST_DIR/repo.XXXXXXXXXX)" - cp -R "$TEST_DIR"/tests/test-data/* "$repo" - ( - cd "$repo"; - git init -q .; - git config user.email "[email protected]" - git config user.name "Me me" - git lfs track '*.testtrace' > /dev/null; - git add .; - git commit -q -a -m 'initial'; - ) - echo $repo -} - -destroy_repo() { - [ -d "$1"/.git ] && rm -rf "$1" -} - assert() { if ! $1; then echo "Assertion failed: \"$1\"" @@ -32,22 +13,22 @@ assert() { run_tracie() { # Run tests for the .testtrace types, using the "gl-test-device" and "vk-test-device" device names. - DEVICE_NAME=gl-test-device CI_PROJECT_DIR="$TEST_DIR" \ - "$TEST_DIR/tracie.sh" "$TEST_DIR/tests/traces.yml" testtrace && \ - DEVICE_NAME=vk-test-device CI_PROJECT_DIR="$TEST_DIR" \ - "$TEST_DIR/tracie.sh" "$TEST_DIR/tests/traces.yml" testtrace + python3 $TEST_DIR/tracie.py --file $TEST_DIR/tests/traces.yml --device-name gl-test-device && \ + python3 $TEST_DIR/tracie.py --file $TEST_DIR/tests/traces.yml --device-name vk-test-device } cleanup() { - rm -rf "$TEST_DIR" + [ "$TEST_DIR" = "/tmp/*" ] && rm -rf "$TEST_DIR" } prepare_for_run() { TEST_DIR="$(mktemp -d -t tracie.test.XXXXXXXXXX)" - # Copy all the tracie scripts to the test dir and later make that the - # CI_PROJECT_DIR for the run-tests.sh script. This avoids polluting the - # normal working dir with test result artifacts. + # Copy all the tracie scripts to the test dir for the run-tests.sh script. + # This avoids polluting the normal working dir with test result artifacts. cp -R "$TRACIE_DIR"/. "$TEST_DIR" + cd "$TEST_DIR" + mkdir traces-db + mv tests/test-data/* traces-db/. trap cleanup EXIT # Ensure we have a clean environment. unset TRACIE_STORE_IMAGES @@ -76,89 +57,41 @@ run_test() { } tracie_succeeds_if_all_images_match() { - repo="$(create_repo)" - cd "$repo" - run_tracie assert "[ $? = 0 ]" - - destroy_repo "$repo" } tracie_fails_on_image_mismatch() { - repo="$(create_repo)" - cd "$repo" - sed -i 's/5efda83854befe0155ff8517a58d5b51/8e0a801367e1714463475a824dab363b/g' \ "$TEST_DIR/tests/traces.yml" run_tracie assert "[ $? != 0 ]" - - destroy_repo "$repo" -} - -tracie_ignores_unspecified_trace_types() { - repo="$(create_repo)" - cd "$repo" - - echo " - path: trace1/empty.trace" >> "$TEST_DIR/tests/traces.yml" - echo " expectations:" >> "$TEST_DIR/tests/traces.yml" - echo " - device: gl-test-device" >> "$TEST_DIR/tests/traces.yml" - echo " checksum: 000000000000000" >> "$TEST_DIR/tests/traces.yml" - # For the tests we only scan for the .testtrace type, - # so the .trace file added below should be ignored. - echo "empty" > trace1/empty.trace - git lfs track '*.trace' - git add trace1 - git commit -a -m 'break' - - run_tracie - assert "[ $? = 0 ]" - - destroy_repo "$repo" } tracie_skips_traces_without_checksum() { - repo="$(create_repo)" - cd "$repo" - echo " - path: trace1/red.testtrace" >> "$TEST_DIR/tests/traces.yml" echo " expectations:" >> "$TEST_DIR/tests/traces.yml" echo " - device: bla" >> "$TEST_DIR/tests/traces.yml" echo " checksum: 000000000000000" >> "$TEST_DIR/tests/traces.yml" # red.testtrace should be skipped, since it doesn't # have any checksums for our device - echo "ff0000ff" > trace1/red.testtrace - git add trace1 - git commit -a -m 'red' + echo "ff0000ff" > traces-db/trace1/red.testtrace run_tracie assert "[ $? = 0 ]" - - destroy_repo "$repo" } tracie_fails_on_dump_image_error() { - repo="$(create_repo)" - cd "$repo" - # "invalid" should fail to parse as rgba and # cause an error - echo "invalid" > trace1/magenta.testtrace - git add trace1 - git commit -a -m 'invalid' + echo "invalid" > traces-db/trace1/magenta.testtrace run_tracie assert "[ $? != 0 ]" - - destroy_repo "$repo" } tracie_stores_only_logs_on_checksum_match() { - repo="$(create_repo)" - cd "$repo" - run_tracie assert "[ $? = 0 ]" @@ -169,14 +102,9 @@ tracie_stores_only_logs_on_checksum_match() { assert "[ ! -f "$TEST_DIR/results/trace2/test/vk-test-device/olive.testtrace-0.png" ]" ls -lR "$TEST_DIR" - - destroy_repo "$repo" } tracie_stores_images_on_checksum_mismatch() { - repo="$(create_repo)" - cd "$repo" - sed -i 's/5efda83854befe0155ff8517a58d5b51/8e0a801367e1714463475a824dab363b/g' \ "$TEST_DIR/tests/traces.yml" @@ -185,14 +113,9 @@ tracie_stores_images_on_checksum_mismatch() { assert "[ ! -f "$TEST_DIR/results/trace1/test/gl-test-device/magenta.testtrace-0.png" ]" assert "[ -f "$TEST_DIR/results/trace2/test/vk-test-device/olive.testtrace-0.png" ]" - - destroy_repo "$repo" } tracie_stores_images_on_request() { - repo="$(create_repo)" - cd "$repo" - (export TRACIE_STORE_IMAGES=1; run_tracie) assert "[ $? = 0 ]" @@ -200,13 +123,10 @@ tracie_stores_images_on_request() { assert "[ -f "$TEST_DIR/results/trace2/test/vk-test-device/olive.testtrace-0.png" ]" ls -lR "$TEST_DIR" - - destroy_repo "$repo" } run_test tracie_succeeds_if_all_images_match run_test tracie_fails_on_image_mismatch -run_test tracie_ignores_unspecified_trace_types run_test tracie_skips_traces_without_checksum run_test tracie_fails_on_dump_image_error run_test tracie_stores_only_logs_on_checksum_match diff --git a/.gitlab-ci/tracie/tracie.py b/.gitlab-ci/tracie/tracie.py new file mode 100644 index 00000000000..445f566f200 --- /dev/null +++ b/.gitlab-ci/tracie/tracie.py @@ -0,0 +1,167 @@ +import argparse +import enum +import glob +import hashlib +import os +import requests +import sys +import tempfile +import time +import yaml + +from pathlib import Path +from PIL import Image +from urllib import parse + +import dump_trace_images + +TRACES_DB_PATH = os.getcwd() + "/traces-db/" +RESULTS_PATH = os.getcwd() + "/results/" + +def replay(trace_path, device_name): + success = dump_trace_images.dump_from_trace(trace_path, [], device_name) + + if not success: + print("[check_image] Trace %s couldn't be replayed. See above logs for more information." % (str(trace_path))) + return None, None, None + else: + base_path = trace_path.parent + file_name = trace_path.name + files = glob.glob(str(base_path / "test" / device_name / (file_name + "-*" + ".png"))) + assert(files) + image_file = files[0] + files = glob.glob(str(base_path / "test" / device_name / (file_name + ".log"))) + assert(files) + log_file = files[0] + return hashlib.md5(Image.open(image_file).tobytes()).hexdigest(), image_file, log_file + +def download_metadata(repo_url, repo_commit, trace_path): + # The GitLab API doesn't want the .git postfix + url = repo_url + if url.endswith(".git"): + url = url[:-4] + url = parse.urlparse(url) + + url_path = url.path + if url_path.startswith("/"): + url_path = url_path[1:] + + gitlab_api_url = url.scheme + "://" + url.netloc + "/api/v4/projects/" + parse.quote_plus(url_path) + + r = requests.get(gitlab_api_url + "/repository/files/%s/raw?ref=%s" % (parse.quote_plus(trace_path), repo_commit)) + metadata_raw = r.text.strip().split('\n') + metadata = dict(line.split(' ', 1) for line in metadata_raw[1:]) + oid = metadata["oid"][7:] if metadata["oid"].startswith('sha256:') else metadata["oid"] + size = int(metadata['size']) + + return oid, size + +def download_trace(repo_url, repo_commit, trace_path, oid, size): + headers = { + "Accept": "application/vnd.git-lfs+json", + "Content-Type": "application/vnd.git-lfs+json" + } + json = { + "operation": "download", + "transfers": [ "basic" ], + "ref": { "name": "refs/heads/%s" % repo_commit }, + "objects": [ + { + "oid": oid, + "size": size + } + ] + } + + # The LFS API really wants the .git postfix... + if not repo_url.endswith(".git"): + repo_url += ".git" + + r = requests.post(repo_url + "/info/lfs/objects/batch", headers=headers, json=json) + url = r.json()["objects"][0]["actions"]["download"]["href"] + open(TRACES_DB_PATH + trace_path, "wb").write(requests.get(url).content) + +def checksum(filename, hash_factory=hashlib.sha256, chunk_num_blocks=128): + h = hash_factory() + with open(filename,'rb') as f: + for chunk in iter(lambda: f.read(chunk_num_blocks*h.block_size), b''): + h.update(chunk) + return h.digest() + +def ensure_trace(repo_url, repo_commit, trace): + trace_path = TRACES_DB_PATH + trace['path'] + if repo_url is None: + assert(repo_commit is None) + assert(os.path.exists(trace_path)) + return + + os.makedirs(os.path.dirname(trace_path), exist_ok=True) + + if os.path.exists(trace_path): + local_oid = checksum(trace_path) + + remote_oid, size = download_metadata(repo_url, repo_commit, trace['path']) + + if not os.path.exists(trace_path) or local_oid != remote_oid: + print("[check_image] Downloading trace %s" % (trace['path']), end=" ", flush=True) + download_time = time.time() + download_trace(repo_url, repo_commit, trace['path'], remote_oid, size) + print("took %ds." % (time.time() - download_time), flush=True) + +def check_trace(repo_url, repo_commit, device_name, trace, expectation): + ensure_trace(repo_url, repo_commit, trace) + + trace_path = Path(TRACES_DB_PATH + trace['path']) + checksum, image_file, log_file = replay(trace_path, device_name) + if checksum is None: + return False + elif checksum == expectation['checksum']: + print("[check_image] Images match for %s" % (trace['path'])) + ok = True + else: + print("[check_image] Images differ for %s (expected: %s, actual: %s)" % + (trace['path'], expectation['checksum'], checksum)) + print("[check_image] For more information see " + "https://gitlab.freedesktop.org/mesa/mesa/blob/master/.gitlab-ci/tracie/README.md") + ok = False + + trace_dir = os.path.split(trace['path'])[0] + results_path = os.path.join(RESULTS_PATH, trace_dir, "test", device_name) + os.makedirs(results_path, exist_ok=True) + os.rename(log_file, os.path.join(results_path, os.path.split(log_file)[1])) + if not ok or os.environ.get('TRACIE_STORE_IMAGES', '0') == '1': + os.rename(image_file, os.path.join(results_path, os.path.split(image_file)[1])) + + return ok + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--file', required=True, + help='the name of the traces.yml file listing traces and their checksums for each device') + parser.add_argument('--device-name', required=True, + help="the name of the graphics device used to replay traces") + + args = parser.parse_args() + + with open(args.file, 'r') as f: + y = yaml.safe_load(f) + + if "traces-db" in y: + repo = y["traces-db"]["repo"] + commit_id = y["traces-db"]["commit"] + else: + repo = None + commit_id = None + + traces = y['traces'] + all_ok = True + for trace in traces: + for expectation in trace['expectations']: + if expectation['device'] == args.device_name: + ok = check_trace(repo, commit_id, args.device_name, trace, expectation) + all_ok = all_ok and ok + + sys.exit(0 if all_ok else 1) + +if __name__ == "__main__": + main() diff --git a/.gitlab-ci/tracie/tracie.sh b/.gitlab-ci/tracie/tracie.sh deleted file mode 100755 index afae5be3365..00000000000 --- a/.gitlab-ci/tracie/tracie.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash - -TRACIE_SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" -TRACES_YAML="$(readlink -f "$1")" -TRACE_TYPE="$2" - -# Clone the traces-db repo without a checkout. Since we are dealing with -# git-lfs repositories, such clones are very lightweight. We check out -# individual files as needed at a later stage (see fetch_trace). -clone_traces_db_no_checkout() -{ - local repo="$1" - local commit="$2" - rm -rf traces-db - git clone --no-checkout -c lfs.storage="$CI_PROJECT_DIR/.git-lfs-storage" "$repo" traces-db - (cd traces-db; git reset "$commit" || git reset "origin/$commit") -} - -query_traces_yaml() -{ - python3 "$TRACIE_SCRIPT_DIR/query_traces_yaml.py" \ - --file "$TRACES_YAML" "$@" -} - -create_clean_git() -{ - rm -rf .clean_git - cp -R .git .clean_git -} - -restore_clean_git() -{ - rm -rf .git - cp -R .clean_git .git -} - -fetch_trace() -{ - local trace="${1//,/?}" - echo -n "[fetch_trace] Fetching $1... " - local output=$(git lfs pull -I "$trace" 2>&1) - local ret=0 - if [[ $? -ne 0 || ! -f "$1" ]]; then - echo "ERROR" - echo "$output" - ret=1 - else - echo "OK" - fi - # Restore a clean .git directory, effectively removing any downloaded - # git-lfs objects, in order to limit required storage. Note that the - # checked out trace file is still present at this point. We remove it - # when we are done with the trace replay at a later stage. - restore_clean_git - return $ret -} - -get_dumped_file() -{ - local trace="$1" - local tracedir="$(dirname "$trace")" - local tracename="$(basename "$trace")" - - find "$tracedir/test/$DEVICE_NAME" -name "$tracename*.$2" -} - -check_image() -{ - local trace="$1" - local image="$2" - - checksum=$(python3 "$TRACIE_SCRIPT_DIR/image_checksum.py" "$image") - expected=$(query_traces_yaml checksum --device-name "$DEVICE_NAME" "$trace") - if [[ "$checksum" = "$expected" ]]; then - echo "[check_image] Images match for $trace" - return 0 - else - echo "[check_image] Images differ for $trace (expected: $expected, actual: $checksum)" - echo "[check_image] For more information see https://gitlab.freedesktop.org/mesa/mesa/blob/master/.gitlab-ci/tracie/README.md" - return 1 - fi -} - -archive_artifact() -{ - mkdir -p "$CI_PROJECT_DIR/results" - cp --parents "$1" "$CI_PROJECT_DIR/results" -} - -if [[ -n "$(query_traces_yaml traces_db_repo)" ]]; then - clone_traces_db_no_checkout "$(query_traces_yaml traces_db_repo)" \ - "$(query_traces_yaml traces_db_commit)" - cd traces-db -else - echo "Warning: No traces-db entry in $TRACES_YAML, assuming traces-db is current directory" -fi - -# During git operations various git objects get created which -# may take up significant space. Store a clean .git instance, -# which we restore after various git operations to keep our -# storage consumption low. -create_clean_git - -ret=0 - -for trace in $(query_traces_yaml traces --device-name "$DEVICE_NAME" --trace-types "$TRACE_TYPE") -do - [[ -n "$(query_traces_yaml checksum --device-name "$DEVICE_NAME" "$trace")" ]] || - { echo "[fetch_trace] Skipping $trace since it has no checksums for $DEVICE_NAME"; continue; } - fetch_trace "$trace" || exit $? - python3 "$TRACIE_SCRIPT_DIR/dump_trace_images.py" --device-name "$DEVICE_NAME" "$trace" || exit $? - image="$(get_dumped_file "$trace" png)" - check_image "$trace" "$image" && check_succeeded=true || { ret=1; check_succeeded=false; } - if [[ "$check_succeeded" = false || "$TRACIE_STORE_IMAGES" = "1" ]]; then - archive_artifact "$image" - fi - archive_artifact "$(get_dumped_file "$trace" log)" - # Remove the downloaded trace file to reduce the total amount of storage - # that is required. - rm "$trace" -done - -exit $ret |