gitlab-ci: More stable URL for kernel and ramdisk artifacts, for LAVA

Place the kernel and ramdisk into a place in the file server so the URL
will only change when the contents also change.

Also put the Mesa build into a separate tarball so the ramdisk's
contents don't change every build.

With proper caching in place, all devices in the same farm need only to
download the mesa tarball once, saving time.

As we switch to MinIO for making kernels and rootfs available to LAVA
devices, we can stop using Docker to distribute them.

Instead, build when needed in separate jobs that push directly to MinIO,
from where LAVA devices can download them.

Signed-off-by: Tomeu Vizoso <[email protected]>
Acked-by: Daniel Stone <[email protected]>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/5515>

1 files changed, 69 insertions, 46 deletions

diff --git a/.gitlab-ci/container/lava_arm.sh b/.gitlab-ci/container/lava_arm.sh
index e5da199abc8..ec3bf52b9f4 100644..100755
--- a/.gitlab-ci/container/lava_arm.sh
+++ b/.gitlab-ci/container/lava_arm.sh
@@ -3,6 +3,15 @@
 set -e
 set -o xtrace
 
+# If remote files are up-to-date, skip rebuilding them
+MINIO_PATH=minio-packet.freedesktop.org/mesa-lava/${CI_PROJECT_PATH}/${DISTRIBUTION_TAG}/${DEBIAN_ARCH}
+DONE_FILE=https://${MINIO_PATH}/done
+if wget -q --method=HEAD ${DONE_FILE}; then
+    exit
+fi
+
+. .gitlab-ci/container/container_pre_build.sh
+
 if [[ "$DEBIAN_ARCH" = "arm64" ]]; then
     GCC_ARCH="aarch64-linux-gnu"
     KERNEL_ARCH="arm64"
@@ -15,6 +24,7 @@ else
     DEFCONFIG="arch/arm/configs/multi_v7_defconfig"
     DEVICE_TREES="arch/arm/boot/dts/rk3288-veyron-jaq.dtb arch/arm/boot/dts/sun8i-h3-libretech-all-h3-cc.dtb"
     KERNEL_IMAGE_NAME="zImage"
+    . .gitlab-ci/create-cross-file.sh armhf
 fi
 
 # Determine if we're in a cross build.
@@ -26,6 +36,46 @@ if [[ -e /cross_file-$DEBIAN_ARCH.txt ]]; then
     export CROSS_COMPILE="${GCC_ARCH}-"
 fi
 
+apt-get update
+apt-get install -y automake \
+                   git \
+                   bc \
+                   cmake \
+                   cpio \
+                   wget \
+                   debootstrap \
+                   libboost-dev \
+                   libegl1-mesa-dev \
+                   libgbm-dev \
+                   libgles2-mesa-dev \
+                   libpcre3-dev \
+                   libpng-dev \
+                   libpython3-dev \
+                   libssl-dev \
+                   libvulkan-dev \
+                   libxcb-keysyms1-dev \
+                   python3-dev \
+                   python3-distutils \
+                   python3-serial \
+                   qt5-default \
+                   qt5-qmake \
+                   qtbase5-dev
+
+
+if [[ "$DEBIAN_ARCH" = "armhf" ]]; then
+	apt-get install -y libboost-dev:armhf \
+		libegl1-mesa-dev:armhf \
+		libelf-dev:armhf \
+		libgbm-dev:armhf \
+		libgles2-mesa-dev:armhf \
+		libpcre3-dev:armhf \
+		libpng-dev:armhf \
+		libpython3-dev:armhf \
+		libvulkan-dev:armhf \
+		libxcb-keysyms1-dev:armhf \
+               qtbase5-dev:armhf
+fi
+
 ############### Build dEQP runner
 . .gitlab-ci/build-cts-runner.sh
 mkdir -p /lava-files/rootfs-${DEBIAN_ARCH}/usr/bin
@@ -63,6 +113,7 @@ rm -rf /renderdoc
 
 
 ############### Cross-build kernel
+KERNEL_URL="https://gitlab.freedesktop.org/tomeu/linux/-/archive/v5.5-panfrost-fixes/linux-v5.5-panfrost-fixes.tar.gz"
 mkdir -p kernel
 wget -qO- ${KERNEL_URL} | tar -xz --strip-components=1 -C kernel
 pushd kernel
@@ -73,19 +124,6 @@ for image in ${KERNEL_IMAGE_NAME}; do
 done
 cp ${DEVICE_TREES} /lava-files/.
 
-
-if [[ ${DEBIAN_ARCH} = "arm64" ]] && which mkimage > /dev/null; then
-    make Image.lzma
-    mkimage \
-        -f auto \
-        -A arm \
-        -O linux \
-        -d arch/arm64/boot/Image.lzma \
-        -C lzma\
-        -b arch/arm64/boot/dts/qcom/sdm845-cheza-r3.dtb \
-        /lava-files/cheza-kernel
-fi
-
 popd
 rm -rf kernel
 
@@ -105,37 +143,22 @@ set -e
 cp .gitlab-ci/create-rootfs.sh /lava-files/rootfs-${DEBIAN_ARCH}/.
 chroot /lava-files/rootfs-${DEBIAN_ARCH} sh /create-rootfs.sh
 rm /lava-files/rootfs-${DEBIAN_ARCH}/create-rootfs.sh
+pushd /lava-files/rootfs-${DEBIAN_ARCH}
+  find -H  |  cpio -H newc -o | gzip -c - > /lava-files/lava-rootfs.cpio.gz
+popd
+rm -rf /lava-files/rootfs-${DEBIAN_ARCH}
+
+ls -lh /lava-files/
+
+. .gitlab-ci/container/container_post_build.sh
+
+############### Upload the files!
+ci-fairy minio login $CI_JOB_JWT
+for f in $(ls /lava-files/); do
+    ci-fairy minio cp /lava-files/$f \
+        minio://${MINIO_PATH}/$f
+done
+
+touch /lava-files/done
+ci-fairy minio cp /lava-files/done minio://${MINIO_PATH}/done
 
-if [ ${DEBIAN_ARCH} = arm64 ]; then
-    # Pull down a specific build of qcomlt/release/qcomlt-5.4 8c79b3d12355
-    # ("Merge tag 'v5.4.23' into release/qcomlt-5.4"), where I used the
-    # .config from
-    # http://snapshots.linaro.org/96boards/dragonboard820c/linaro/debian/457/config-5.4.0-qcomlt-arm64
-    # with the following merged in:
-    #
-    # CONFIG_DRM=y
-    # CONFIG_DRM_MSM=y
-    # CONFIG_ATL1C=y
-    #
-    # Reason: 5.5 has a big stack of oopses and warns on db820c.  4.14-5.4
-    # linaro kernel binaries (see above .config link) have these as modules
-    # and distributed the modules only in the debian system, not the initrd,
-    # so they're very hard to extract (involving simg2img and loopback
-    # mounting).  4.11 is missing d72fea538fe6 ("drm/msm: Fix the check for
-    # the command size") so it can't actually run fredreno.  qcomlt-4.14 is
-    # unstable at boot (~10% instaboot rate).  The 5.4 qcomlt kernel with msm
-    # built in seems like the easiest way to go.
-    wget https://people.freedesktop.org/~anholt/qcomlt-5.4-msm-build/Image.gz -O Image.gz \
-         -O /lava-files/db820c-kernel
-    wget https://people.freedesktop.org/~anholt/qcomlt-5.4-msm-build/apq8096-db820c.dtb \
-         -O /lava-files/db820c.dtb
-
-    # Make a gzipped copy of the Image for db410c.
-    gzip -k /lava-files/Image
-
-    # Add missing a630 firmware, added to debian packge in apr 2020
-    wget https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/plain/qcom/a630_gmu.bin \
-         -O /lava-files/rootfs-arm64/lib/firmware/qcom/a630_gmu.bin
-    wget https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/plain/qcom/a630_sqe.fw \
-         -O /lava-files/rootfs-arm64/lib/firmware/qcom/a630_sqe.fw
-fi