ci: Include trace replay support in ARM rootfses.

Builds the renderdoc and apitrace programs so we can replay GL traces on
DUTs.

[Separated out from 5472's commit that also enabled the jobs in LAVA,
dropped unnecessary python packages from arm_build, fixed up arm64_test
build, traces-db in baremetal, new commit message by anholt]

Signed-off-by: Rohan Garg <[email protected]>
Signed-off-by: Tomeu Vizoso <[email protected]>
Reviewed-by: Eric Anholt <[email protected]>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/5433>

9 files changed, 105 insertions, 13 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 59ebcf4015a..a59cfcd4458 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -272,7 +272,7 @@ arm_build:
     - .fdo.container-build@debian@arm64v8
     - .container
   variables:
-    FDO_DISTRIBUTION_TAG: &arm_build "2020-06-22-stable"
+    FDO_DISTRIBUTION_TAG: &arm_build "2020-06-22-tracie"
 
 .use-arm_build:
   variables:
@@ -287,7 +287,7 @@ arm_test-base:
     - .fdo.container-build@debian
     - .container
   variables:
-    FDO_DISTRIBUTION_TAG: &arm_test-base "2020-06-22-stable"
+    FDO_DISTRIBUTION_TAG: &arm_test-base "2020-06-22-tracie"
 
 .use-arm_test-base:
   extends:
@@ -305,7 +305,7 @@ arm64_test:
   extends:
     - .use-arm_test-base
   variables:
-    FDO_DISTRIBUTION_TAG: &arm64_test "2020-06-22-ephemeral"
+    FDO_DISTRIBUTION_TAG: &arm64_test "2020-06-22-tracie"
 
 .use-arm64_test:
   variables:
diff --git a/.gitlab-ci/bare-metal/fastboot.sh b/.gitlab-ci/bare-metal/fastboot.sh
index 9a4e6f64959..ad4a665055b 100755
--- a/.gitlab-ci/bare-metal/fastboot.sh
+++ b/.gitlab-ci/bare-metal/fastboot.sh
@@ -60,7 +60,8 @@ rsync -a --delete $BM_ROOTFS/ rootfs/
 # these devices use it and it would take up space in the initrd.
 pushd rootfs
 find -H | \
-  egrep -v "external/(openglcts|vulkancts|amber|glslang|spirv-tools)" | \
+  egrep -v "external/(openglcts|vulkancts|amber|glslang|spirv-tools)" |
+  egrep -v "traces-db|apitrace|renderdoc|python" | \
   cpio -H newc -o | \
   xz --check=crc32 -T4 - > $CI_PROJECT_DIR/rootfs.cpio.gz
 popd
diff --git a/.gitlab-ci/build-apitrace.sh b/.gitlab-ci/build-apitrace.sh
index 87582f3b9e6..8b44b66a2ce 100644
--- a/.gitlab-ci/build-apitrace.sh
+++ b/.gitlab-ci/build-apitrace.sh
@@ -9,22 +9,25 @@ WAFFLE_VERSION="e3c995d9a2693b687501715b6550619922346089"
 git clone https://gitlab.freedesktop.org/mesa/waffle.git --single-branch --no-checkout /waffle
 pushd /waffle
 git checkout "$WAFFLE_VERSION"
-cmake -B_build -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_BUILD_TYPE=Release .
-make -C _build -j4 install
+cmake -B_build -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_BUILD_TYPE=Release $EXTRA_CMAKE_ARGS .
+make -C _build install
+mkdir -p build/lib build/bin
+cp _build/lib/libwaffle-1.so build/lib/libwaffle-1.so.0
+cp _build/bin/wflinfo build/bin/wflinfo
+${STRIP_CMD:-strip} build/lib/* build/bin/*
+find . -not -path './build' -not -path './build/*' -delete
 popd
-rm -rf /waffle
 
 APITRACE_VERSION="9.0"
 
 git clone https://github.com/apitrace/apitrace.git --single-branch --no-checkout /apitrace
 pushd /apitrace
 git checkout "$APITRACE_VERSION"
-cmake -G Ninja -B_build -H. -DCMAKE_BUILD_TYPE=Release -DENABLE_GUI=False -DENABLE_WAFFLE=on -DWaffle_DIR=/usr/local/lib/cmake/Waffle/
+cmake -G Ninja -B_build -H. -DCMAKE_BUILD_TYPE=Release -DENABLE_GUI=False -DENABLE_WAFFLE=on -DWaffle_DIR=/usr/local/lib/cmake/Waffle/ $EXTRA_CMAKE_ARGS
 ninja -C _build
 mkdir build
 cp _build/apitrace build
-cp _build/glretrace build
 cp _build/eglretrace build
-strip build/*
+${STRIP_CMD:-strip} build/*
 find . -not -path './build' -not -path './build/*' -delete
 popd
diff --git a/.gitlab-ci/build-renderdoc.sh b/.gitlab-ci/build-renderdoc.sh
index 5d023c73627..bc21cc864a9 100644
--- a/.gitlab-ci/build-renderdoc.sh
+++ b/.gitlab-ci/build-renderdoc.sh
@@ -7,11 +7,11 @@ RENDERDOC_VERSION=da02e88201dc3b64316fc33ce6ff69cc729689aa
 git clone https://github.com/baldurk/renderdoc.git --single-branch --no-checkout /renderdoc
 pushd /renderdoc
 git checkout "$RENDERDOC_VERSION"
-cmake -G Ninja -B_build -H. -DENABLE_QRENDERDOC=false -DCMAKE_BUILD_TYPE=Release
+cmake -G Ninja -B_build -H. -DENABLE_QRENDERDOC=false -DCMAKE_BUILD_TYPE=Release $EXTRA_CMAKE_ARGS
 ninja -C _build
 mkdir -p build/lib
+${STRIP_CMD:-strip} _build/lib/*.so
 cp _build/lib/renderdoc.so build/lib
 cp _build/lib/librenderdoc.so build/lib
-strip build/lib/*
 find . -not -path './build' -not -path './build/*' -delete
 popd
diff --git a/.gitlab-ci/container/arm_build.sh b/.gitlab-ci/container/arm_build.sh
index 6facab31e73..4266aee2ba8 100644
--- a/.gitlab-ci/container/arm_build.sh
+++ b/.gitlab-ci/container/arm_build.sh
@@ -12,6 +12,8 @@ apt-get update
 apt-get -y install \
 	abootimg \
 	android-sdk-ext4-utils \
+	autoconf \
+	automake \
 	bc \
 	bison \
 	ccache \
@@ -24,28 +26,45 @@ apt-get -y install \
 	g++ \
 	git \
 	lavacli \
+	libboost-dev:armhf \
+	libboost-dev \
 	libdrm-dev:armhf \
+	libdrm-dev \
 	libegl1-mesa-dev \
 	libegl1-mesa-dev:armhf \
 	libelf-dev \
 	libelf-dev:armhf \
 	libexpat1-dev \
 	libexpat1-dev:armhf \
+	libgbm-dev \
+	libgbm-dev:armhf \
 	libgles2-mesa-dev \
 	libgles2-mesa-dev:armhf \
+	libpcre3-dev \
+	libpcre3-dev:armhf \
 	libpng-dev \
 	libpng-dev:armhf \
+	libpython3-dev \
+	libpython3-dev:armhf \
 	libssl-dev \
 	libvulkan-dev \
+	libvulkan-dev \
 	libvulkan-dev:armhf \
+	libxcb-keysyms1-dev \
+	libxcb-keysyms1-dev:armhf \
 	llvm-7-dev:armhf \
 	llvm-8-dev \
 	pkg-config \
 	python \
+	python3-dev \
 	python3-distutils \
 	python3-setuptools \
 	python3-mako \
 	python3-serial \
+	qt5-default \
+	qt5-qmake \
+	qtbase5-dev \
+	qtbase5-dev:armhf \
 	unzip \
 	wget \
 	xz-utils \
diff --git a/.gitlab-ci/container/baremetal_build.sh b/.gitlab-ci/container/baremetal_build.sh
index b84608f87e9..6024138f8ab 100644
--- a/.gitlab-ci/container/baremetal_build.sh
+++ b/.gitlab-ci/container/baremetal_build.sh
@@ -3,24 +3,38 @@
 set -e
 set -o xtrace
 
+ROOTFS=/lava-files/rootfs-${arch}
+
 dpkg --add-architecture $arch
 apt-get update
 
 # Cross-build test deps
 BAREMETAL_EPHEMERAL=" \
+        autoconf \
+        automake \
         crossbuild-essential-$arch \
+        git-lfs \
         libdrm-dev:$arch \
+        libboost-dev:$arch \
         libegl1-mesa-dev:$arch \
         libelf-dev:$arch \
         libexpat1-dev:$arch \
         libffi-dev:$arch \
         libgbm-dev:$arch \
         libgles2-mesa-dev:$arch \
+        libpcre3-dev:$arch \
         libpng-dev:$arch \
+        libpython3-dev:$arch \
         libstdc++6:$arch \
         libtinfo-dev:$arch \
         libegl1-mesa-dev:$arch \
-        libvulkan-dev:$arch
+        libvulkan-dev:$arch \
+        libxcb-keysyms1-dev:$arch \
+        libpython3-dev:$arch \
+        python3-dev \
+        qt5-default \
+        qt5-qmake \
+        qtbase5-dev:$arch \
         "
 
 apt-get install -y --no-remove $BAREMETAL_EPHEMERAL
@@ -37,6 +51,20 @@ mkdir /var/cache/apt/archives/$arch
 
 DEBIAN_ARCH=$arch INCLUDE_VK_CTS=1 . .gitlab-ci/container/lava_arm.sh
 
+############### Store traces
+# Clone the traces-db at container build time so we don't have to pull traces
+# per run (too much egress cost for fd.o).
+git clone \
+    --depth 1 \
+    -b mesa-ci-2020-06-08 \
+    https://gitlab.freedesktop.org/gfx-ci/tracie/traces-db.git \
+    $ROOTFS/traces-db
+rm -rf $ROOTFS/traces-db/.git
+find $ROOTFS/traces-db -type f \
+     -a -not -name '*.trace' \
+     -a -not -name '*.rdc' \
+     -delete
+
 ccache --show-stats
 
 . .gitlab-ci/container/container_post_build.sh
diff --git a/.gitlab-ci/container/lava_arm.sh b/.gitlab-ci/container/lava_arm.sh
index 05bc602aa6f..ed1a10eaec5 100644
--- a/.gitlab-ci/container/lava_arm.sh
+++ b/.gitlab-ci/container/lava_arm.sh
@@ -43,6 +43,25 @@ fi
 mv /deqp /lava-files/rootfs-${DEBIAN_ARCH}/.
 
 
+############### Build apitrace
+. .gitlab-ci/build-apitrace.sh
+mkdir -p /lava-files/rootfs-${DEBIAN_ARCH}/apitrace
+mv /apitrace/build /lava-files/rootfs-${DEBIAN_ARCH}/apitrace
+rm -rf /apitrace
+
+mkdir -p /lava-files/rootfs-${DEBIAN_ARCH}/waffle
+mv /waffle/build /lava-files/rootfs-${DEBIAN_ARCH}/waffle
+rm -rf /waffle
+
+
+############### Build renderdoc
+EXTRA_CMAKE_ARGS+=" -DENABLE_XCB=false"
+. .gitlab-ci/build-renderdoc.sh
+mkdir -p /lava-files/rootfs-${DEBIAN_ARCH}/renderdoc
+mv /renderdoc/build /lava-files/rootfs-${DEBIAN_ARCH}/renderdoc
+rm -rf /renderdoc
+
+
 ############### Cross-build kernel
 KERNEL_URL="https://gitlab.freedesktop.org/tomeu/linux/-/archive/v5.5-panfrost-fixes/linux-v5.5-panfrost-fixes.tar.gz"
 
diff --git a/.gitlab-ci/create-rootfs.sh b/.gitlab-ci/create-rootfs.sh
index 46a3dd239df..ef1026fea7c 100644
--- a/.gitlab-ci/create-rootfs.sh
+++ b/.gitlab-ci/create-rootfs.sh
@@ -11,8 +11,16 @@ apt-get -y install --no-install-recommends \
     libexpat1 \
     libdrm2 \
     libdrm-nouveau2 \
+    libx11-6 \
+    libx11-xcb1 \
     firmware-qcom-media \
     netcat-openbsd \
+    python3 \
+    libpython3.7 \
+    python3-pil \
+    python3-pytest \
+    python3-requests \
+    python3-yaml \
     wget \
     xz-utils
 
@@ -72,6 +80,7 @@ rm -rf /var/log/*
 # Dropping documentation, localization, i18n files, etc
 rm -rf /usr/share/doc/*
 rm -rf /usr/share/locale/*
+rm -rf /usr/share/X11/locale/*
 rm -rf /usr/share/man
 rm -rf /usr/share/i18n/*
 rm -rf /usr/share/info/*
@@ -115,10 +124,20 @@ UNNEEDED_PACKAGES="apt libapt-pkg6.0 "\
 "passwd "\
 "libsemanage1 libsemanage-common "\
 "libsepol1 "\
+"gzip "\
 "gpgv "\
 "hostname "\
 "adduser "\
 "debian-archive-keyring "\
+"libegl1-mesa-dev "\
+"libegl-mesa0 "\
+"libgl1-mesa-dev "\
+"libgl1-mesa-dri "\
+"libglapi-mesa "\
+"libgles2-mesa-dev "\
+"libglx-mesa0 "\
+"mesa-common-dev "\
+"libz3-4 "\
 
 # Removing unneeded packages
 for PACKAGE in ${UNNEEDED_PACKAGES}
diff --git a/.gitlab-ci/tracie-runner-gl.sh b/.gitlab-ci/tracie-runner-gl.sh
index 426333ce295..5c47114e43f 100755
--- a/.gitlab-ci/tracie-runner-gl.sh
+++ b/.gitlab-ci/tracie-runner-gl.sh
@@ -17,6 +17,9 @@ export LD_LIBRARY_PATH="/waffle/build/lib:$LD_LIBRARY_PATH"
 # Set environment for apitrace executable.
 export PATH="/apitrace/build:$PATH"
 
+# Set environment for wflinfo executable.
+export PATH="/waffle/build/bin:$PATH"
+
 # Use the surfaceless EGL platform.
 export EGL_PLATFORM="surfaceless"
 export DISPLAY=