From 50c83bf6c8d7bdcd5057bff93f37689bb691c5bf Mon Sep 17 00:00:00 2001 From: Daniel Stone Date: Mon, 21 Jun 2021 23:53:51 +0100 Subject: [PATCH] CI: Split OS-independent section from debian-install.sh When we build our base-OS container, we run debian-install.sh to install packages and compile our build dependencies. Since the latter is mostly OS-independent, split debian-install.sh into two scripts: one to install and cleanup packages, and another just to compile stuff. Signed-off-by: Daniel Stone --- .gitlab-ci/build-deps.sh | 126 +++++++++++++++++++++++++++++++++++ .gitlab-ci/debian-install.sh | 122 ++------------------------------- 2 files changed, 131 insertions(+), 117 deletions(-) create mode 100755 .gitlab-ci/build-deps.sh diff --git a/.gitlab-ci/build-deps.sh b/.gitlab-ci/build-deps.sh new file mode 100755 index 00000000..4d9a821e --- /dev/null +++ b/.gitlab-ci/build-deps.sh @@ -0,0 +1,126 @@ +#!/bin/bash +# +# Builds the dependencies required for any OS/architecture combination. See +# .gitlab-ci.yml for more information. This script is called from an +# OS-specific build scripts like debian-install.sh. + +set -o xtrace -o errexit + +# Set concurrency to an appropriate level for our shared runners, falling back +# to the conservative default form before we had this variable. +export MAKEFLAGS="-j${FDO_CI_CONCURRENT:-4}" +export NINJAFLAGS="-j${FDO_CI_CONCURRENT:-4}" + +# Build and install Meson. Generally we want to keep this in sync with what +# we require inside meson.build, however per wayland/weston@bcf37c937a36, +# we use a higher version here +pip3 install --user git+https://github.com/mesonbuild/meson.git@0.57.0 +export PATH=$HOME/.local/bin:$PATH + +# Our docs are built using Sphinx (top-level organisation and final HTML/CSS +# generation), Doxygen (parse structures/functions/comments from source code), +# Breathe (a bridge between Doxygen and Sphinx), and we use the Read the Docs +# theme for the final presentation. +pip3 install sphinx==2.1.0 --user +pip3 install breathe==4.13.0.post0 --user +pip3 install sphinx_rtd_theme==0.4.3 --user + +# Build a Linux kernel for use in testing. We enable the VKMS module so we can +# predictably test the DRM backend in the absence of real hardware. We lock the +# version here so we see predictable results. +git clone --depth=1 --branch=drm-next-2020-06-11-1 https://anongit.freedesktop.org/git/drm/drm.git linux +cd linux +make x86_64_defconfig +make kvmconfig +./scripts/config --enable CONFIG_DRM_VKMS +make oldconfig +make +cd .. +mkdir /weston-virtme +mv linux/arch/x86/boot/bzImage /weston-virtme/bzImage +mv linux/.config /weston-virtme/.config +rm -rf linux + +# Build virtme, a QEMU wrapper: https://github.com/amluto/virtme +# +# virtme makes our lives easier by abstracting handling of the console, +# filesystem, etc, so we can pretend that the VM we execute in is actually +# just a regular container. +# +# The reason why we are using a fork here is that it adds a patch to have the +# --script-dir command line option. With that we can run scripts that are in a +# certain folder when virtme starts, which is necessary in our use case. +# +# The upstream also has some commands that could help us to reach the same +# results: --script-sh and --script-exec. Unfornutately they are not completely +# implemented yet, so we had some trouble to use them and it was becoming +# hackery. +# +git clone https://github.com/ezequielgarcia/virtme +cd virtme +git checkout -b snapshot 69e3cb83b3405edc99fcf9611f50012a4f210f78 +./setup.py install +cd .. + +# Build and install Wayland; keep this version in sync with our dependency +# in meson.build. +git clone --branch 1.18.0 --depth=1 https://gitlab.freedesktop.org/wayland/wayland +cd wayland +git show -s HEAD +mkdir build +cd build +../autogen.sh --disable-documentation +make install +cd ../../ + +# Keep this version in sync with our dependency in meson.build. If you wish to +# raise a MR against custom protocol, please change this reference to clone +# your relevant tree, and make sure you bump $FDO_DISTRIBUTION_TAG. +git clone --branch 1.19 https://gitlab.freedesktop.org/wayland/wayland-protocols +cd wayland-protocols +git show -s HEAD +mkdir build +cd build +../autogen.sh +make install +cd ../../ +rm -rf wayland-protocols + +# Build and install our own version of Mesa. Debian provides a perfectly usable +# Mesa, however llvmpipe's rendering behaviour can change subtly over time. +# This doesn't work for our tests which expect pixel-precise reproduction, so +# we lock it to a set version for more predictability. If you need newer +# features from Mesa then bump this version and $FDO_DISTRIBUTION_TAG, however +# please be prepared for some of the tests to change output, which will need to +# be manually inspected for correctness. +git clone --single-branch --branch 20.3 --shallow-since='2020-12-15' https://gitlab.freedesktop.org/mesa/mesa.git mesa +cd mesa +git checkout -b snapshot mesa-20.3.1 +meson build -Dauto_features=disabled \ + -Dgallium-drivers=swrast -Dvulkan-drivers= -Ddri-drivers= +ninja ${NINJAFLAGS} -C build install +cd .. +rm -rf mesa + +# PipeWire is used for remoting support. Unlike our other dependencies its +# behaviour will be stable, however as a pre-1.0 project its API is not yet +# stable, so again we lock it to a fixed version. +rm -rf pipewire +git clone --single-branch --branch 0.3.31 https://gitlab.freedesktop.org/pipewire/pipewire.git pipewire +cd pipewire +meson build +ninja ${NINJAFLAGS} -C build install +cd .. +rm -rf pipewire + +# seatd lets us avoid the pain of handling VTs manually through weston-launch +# or open-coding TTY assignment within Weston. We use this for our tests using +# the DRM backend. +git clone --depth=1 --branch 0.5.0 https://git.sr.ht/~kennylevinsen/seatd +cd seatd +meson build -Dauto_features=disabled \ + -Dseatd=enabled -Dlogind=enabled -Dserver=enabled \ + -Dexamples=disabled -Dman-pages=disabled +ninja ${NINJAFLAGS} -C build install +cd .. +rm -rf seatd diff --git a/.gitlab-ci/debian-install.sh b/.gitlab-ci/debian-install.sh index bf0edc2c..47e62b56 100644 --- a/.gitlab-ci/debian-install.sh +++ b/.gitlab-ci/debian-install.sh @@ -6,11 +6,6 @@ set -o xtrace -o errexit -# Set concurrency to an appropriate level for our shared runners, falling back -# to the conservative default from before we had this variable. -export MAKEFLAGS="-j${FDO_CI_CONCURRENT:-4}" -export NINJAFLAGS="-j${FDO_CI_CONCURRENT:-4}" - # These get temporary installed for building Linux and then force-removed. LINUX_DEV_PKGS=" bc @@ -98,125 +93,18 @@ apt-get -y --no-install-recommends install \ qemu-system \ sysvinit-core \ xwayland \ + $MESA_DEV_PKGS \ $MESA_RUNTIME_PKGS \ $LINUX_DEV_PKGS \ apt-get -y --no-install-recommends -t buster-backports install \ freerdp2-dev -# Build and install Meson. Generally we want to keep this in sync with what -# we require inside meson.build, however per wayland/weston@bcf37c937a36, -# we use a higher version here -pip3 install --user git+https://github.com/mesonbuild/meson.git@0.57.0 -export PATH=$HOME/.local/bin:$PATH - -# Our docs are built using Sphinx (top-level organisation and final HTML/CSS -# generation), Doxygen (parse structures/functions/comments from source code), -# Breathe (a bridge between Doxygen and Sphinx), and we use the Read the Docs -# theme for the final presentation. -pip3 install sphinx==2.1.0 --user -pip3 install breathe==4.13.0.post0 --user -pip3 install sphinx_rtd_theme==0.4.3 --user - -# Build a Linux kernel for use in testing. We enable the VKMS module so we can -# predictably test the DRM backend in the absence of real hardware. We lock the -# version here so we see predictable results. -git clone --depth=1 --branch=drm-next-2020-06-11-1 https://anongit.freedesktop.org/git/drm/drm.git linux -cd linux -make x86_64_defconfig -make kvmconfig -./scripts/config --enable CONFIG_DRM_VKMS -make oldconfig -make -cd .. -mkdir /weston-virtme -mv linux/arch/x86/boot/bzImage /weston-virtme/bzImage -mv linux/.config /weston-virtme/.config -rm -rf linux - -# Build virtme, a QEMU wrapper: https://github.com/amluto/virtme -# -# virtme makes our lives easier by abstracting handling of the console, -# filesystem, etc, so we can pretend that the VM we execute in is actually -# just a regular container. -# -# The reason why we are using a fork here is that it adds a patch to have the -# --script-dir command line option. With that we can run scripts that are in a -# certain folder when virtme starts, which is necessary in our use case. -# -# The upstream also has some commands that could help us to reach the same -# results: --script-sh and --script-exec. Unfornutately they are not completely -# implemented yet, so we had some trouble to use them and it was becoming -# hackery. -# -git clone https://github.com/ezequielgarcia/virtme -cd virtme -git checkout -b snapshot 69e3cb83b3405edc99fcf9611f50012a4f210f78 -./setup.py install -cd .. - -# Build and install Wayland; keep this version in sync with our dependency -# in meson.build. -git clone --branch 1.18.0 --depth=1 https://gitlab.freedesktop.org/wayland/wayland -cd wayland -git show -s HEAD -mkdir build -cd build -../autogen.sh --disable-documentation -make install -cd ../../ - -# Keep this version in sync with our dependency in meson.build. If you wish to -# raise a MR against custom protocol, please change this reference to clone -# your relevant tree, and make sure you bump $FDO_DISTRIBUTION_TAG. -git clone --branch 1.19 https://gitlab.freedesktop.org/wayland/wayland-protocols -cd wayland-protocols -git show -s HEAD -mkdir build -cd build -../autogen.sh -make install -cd ../../ -rm -rf wayland-protocols - -# Build and install our own version of Mesa. Debian provides a perfectly usable -# Mesa, however llvmpipe's rendering behaviour can change subtly over time. -# This doesn't work for our tests which expect pixel-precise reproduction, so -# we lock it to a set version for more predictability. If you need newer -# features from Mesa then bump this version and $FDO_DISTRIBUTION_TAG, however -# please be prepared for some of the tests to change output, which will need to -# be manually inspected for correctness. -apt-get -y --no-install-recommends install $MESA_DEV_PKGS -git clone --single-branch --branch 20.3 --shallow-since='2020-12-15' https://gitlab.freedesktop.org/mesa/mesa.git mesa -cd mesa -git checkout -b snapshot mesa-20.3.1 -meson build -Dauto_features=disabled \ - -Dgallium-drivers=swrast -Dvulkan-drivers= -Ddri-drivers= -ninja ${NINJAFLAGS} -C build install -cd .. -rm -rf mesa -# PipeWire is used for remoting support. Unlike our other dependencies its -# behaviour will be stable, however as a pre-1.0 project its API is not yet -# stable, so again we lock it to a fixed version. -rm -rf pipewire -git clone --depth=1 --branch 0.3.31 https://gitlab.freedesktop.org/pipewire/pipewire.git pipewire -cd pipewire -meson build -ninja ${NINJAFLAGS} -C build install -cd .. -rm -rf pipewire +# Actually build our dependencies ... +./.gitlab-ci/build-deps.sh -# seatd lets us avoid the pain of handling VTs manually through weston-launch -# or open-coding TTY assignment within Weston. We use this for our tests using -# the DRM backend. -git clone --depth=1 --branch 0.5.0 https://git.sr.ht/~kennylevinsen/seatd -cd seatd -meson build -Dauto_features=disabled \ - -Dseatd=enabled -Dlogind=enabled -Dserver=enabled \ - -Dexamples=disabled -Dman-pages=disabled -ninja ${NINJAFLAGS} -C build install -cd .. -rm -rf seatd +# And remove packages which are only required for our build dependencies, +# which we don't need bloating the image whilst we build and run Weston. apt-get -y --autoremove purge $LINUX_DEV_PKGS $MESA_DEV_PKGS