ci: Switch over to an autoscaling GKE cluster for builds.

The GKE pool we're using is 1-3 32-core VMs, preemptible (to keep
costs down), with 8 jobs concurrent per system.  We have plenty of
memory (4G/core), so we run make -j8 to try to keep the cores busy even
when one job is in a single-threaded step (docker image download, git
clone, artifacts processing, etc.)  When all jobs are generating work
for all the cores, they'll be scheduled fairly.

The nodes in the pool have 300GB boot disks (over-provisioned in space
to provide enough iops and throughput) mounted to /ccache, and
CACHE_DIR set pointing to them.  This means that once a new
autoscaled-up node has run some jobs, it should have a hot ccache from
then on (instead of having to rely on the docker container cache
having our ccache laying around and not getting wiped out by some
other fd.o job).  Local SSDs would provide higher performance, but
unfortunately are not supported with the cluster autoscaler.

For now, the softpipe/llvmpipe test runs are still on the shared
runners, until I can get them ported onto Bas's runner so they can be
parallelized in a single job.

Reviewed-by: Michel Dänzer <[email protected]>

4 files changed, 20 insertions, 8 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 723ccb9889a..bbc38ff156d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -136,19 +136,29 @@ debian-10:arm64:
 # Just Linux
 .build-linux:
   extends: .build-common
+  tags:
+    - mesa-autoscale
   cache:
     key: ${CI_JOB_NAME}
     paths:
       - ccache
   variables:
     CCACHE_COMPILERCHECK: "content"
+    NINJA_FLAGS: "-j8" # autoscale is provisioned at 4 CPUs/job.
+    # kubernetes (the mesa-autoscale runner) always has a clean working
+    # dir, so we can't reuse the git clone, but that also means we don't
+    # need to download any extra history (normally extra history is
+    # needed so that a future fetch can find a shared parent).
+    GIT_DEPTH: 1
+    # Autoscale runners have 300GB of disk (since iops/throughput
+    # scales with disk size!), but we want to leave lots of space for
+    # container caching.
+    CCACHE_SIZE: 20G
   # Use ccache transparently, and print stats before/after
   before_script:
     - export PATH="/usr/lib/ccache:$PATH"
     - export CCACHE_BASEDIR="$PWD"
-    - export CCACHE_DIR="$PWD/ccache"
-    - ccache --max-size=1500M
-    - ccache --zero-stats || true
+    - ccache --max-size="$CCACHE_SIZE" || true
     - ccache --show-stats || true
   after_script:
     # In case the install dir is being saved as artifacts, tar it up
@@ -157,7 +167,6 @@ debian-10:arm64:
     - if [ -d install ]; then
         tar -cf artifacts/install.tar install;
       fi
-    - export CCACHE_DIR="$PWD/ccache"
     - ccache --show-stats
 
 .build-windows:
@@ -181,7 +190,7 @@ debian-10:arm64:
     - .build-linux
     - .use-debian-10:amd64
   variables:
-    SCONSFLAGS: "-j4"
+    SCONSFLAGS: "-j8"
   script:
     - .gitlab-ci/scons-build.sh
 
@@ -243,6 +252,9 @@ meson-main:
     GALLIUM_DRIVERS: "etnaviv,freedreno,kmsro,lima,nouveau,panfrost,swrast,tegra,v3d,vc4"
     EXTRA_OPTION: >
       -D I-love-half-baked-turnips=true
+    NINJA_FLAGS: "-j4"
+    CCACHE_DIR: "$PWD/ccache"
+    CCACHE_SIZE: "1500M"
   needs:
     - debian-10:arm64
   tags:
diff --git a/.gitlab-ci/lava-gitlab-ci.yml b/.gitlab-ci/lava-gitlab-ci.yml
index 89e798a4b0a..f50f07abb24 100644
--- a/.gitlab-ci/lava-gitlab-ci.yml
+++ b/.gitlab-ci/lava-gitlab-ci.yml
@@ -86,7 +86,7 @@ lava-container:arm64:
         -D gallium-xa=false
         -D gallium-nine=false
         -D llvm=false
-    - ninja -C mesa-build -j4
+    - ninja -C mesa-build -j8
     - ninja -C mesa-build install
     - find /artifacts/rootfs/mesa/lib -name \*.so -exec ${GCC_ARCH}-strip {} \;
 
diff --git a/.gitlab-ci/meson-build.sh b/.gitlab-ci/meson-build.sh
index d56e526ef20..0f4f490447d 100755
--- a/.gitlab-ci/meson-build.sh
+++ b/.gitlab-ci/meson-build.sh
@@ -58,7 +58,7 @@ meson _build --native-file=native.file \
       ${EXTRA_OPTION}
 cd _build
 meson configure
-ninja -j4
+ninja ${NINJA_FLAGS}
 LC_ALL=C.UTF-8 ninja test
 ninja install
 cd ..
diff --git a/.gitlab-ci/run-shader-db.sh b/.gitlab-ci/run-shader-db.sh
index 7580457d850..a62a91b5ea9 100755
--- a/.gitlab-ci/run-shader-db.sh
+++ b/.gitlab-ci/run-shader-db.sh
@@ -12,6 +12,6 @@ cd /usr/local/shader-db
 
 for driver in freedreno v3d; do
     env LD_PRELOAD=$LIBDIR/lib${driver}_noop_drm_shim.so \
-        ./run -j 4 ./shaders \
+        ./run -j 8 ./shaders \
             > $ARTIFACTSDIR/${driver}-shader-db.txt
 done