From ebca36da661a8bd62d6d2225db2ddd05c707e827 Mon Sep 17 00:00:00 2001 From: Daniel Stone Date: Mon, 21 Jun 2021 21:34:43 +0100 Subject: [PATCH] CI: Document the build and container process Explain what we do within our CI and why, with links as required. Signed-off-by: Daniel Stone --- .gitlab-ci.yml | 67 +++++++++++++++++++++++++++++++++++- .gitlab-ci/debian-install.sh | 40 +++++++++++++++++++-- 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cce99991..b9f2a054 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,12 +1,54 @@ # vim: set expandtab shiftwidth=2 tabstop=8 textwidth=0: .templates_sha: &template_sha 567700e483aabed992d0a4fea84994a0472deff6 # see https://docs.gitlab.com/ee/ci/yaml/#includefile +# This file uses the freedesktop ci-templates to build Weston and run our +# tests in CI. +# +# ci-templates uses a multi-stage build process. First, the base container +# image is built which contains the core distribution, the toolchain, and +# all our build dependencies. This container is aggressively cached; if a +# container image matching $FDO_DISTRIBUTION_TAG is found in either the +# upstream repo (wayland/weston) or the user's downstream repo, it is +# reused for the build. This gives us predictability of build and far +# quicker runtimes, however it means that any changes to the base container +# must also change $FDO_DISTRIBUTION_TAG. When changing this, please use +# the current date as well as a unique build identifier. +# +# After the container is either rebuilt (tag mismatch) or reused (tag +# previously used), the build stage executes within this container. +# +# The final stage is used to expose documentation and coverage information, +# including publishing documentation to the public site when built on the +# main branch. +# +# Apart from the 'variables', 'include', and 'stages' top-level anchors, +# everything not beginning with a dot ('.') is the name of a job which will +# be executed as part of CI, unless the rules specify that it should not be +# run. +# +# Variables prefixed with CI_ are generally provided by GitLab itself; +# variables prefixed with FDO_ and templates prefixed by .fdo are provided +# by the ci-templates. +# +# For more information on GitLab CI, including the YAML syntax, see: +# https://docs.gitlab.com/ee/ci/yaml/README.html +# +# Note that freedesktop.org uses the 'Community Edition' of GitLab, so features +# marked as 'premium' or 'ultimate' are not available to us. +# +# For more information on ci-templates, see: +# - documentation at https://freedesktop.pages.freedesktop.org/ci-templates/ +# - repo at https://gitlab.freedesktop.org/freedesktop/ci-templates/ variables: FDO_UPSTREAM_REPO: wayland/weston include: + # Here we use a fixed ref in order to isolate ourselves from ci-templates + # API changes. If you need new features from ci-templates you must bump + # this to the current SHA you require from the ci-templates repo, however + # be aware that you may need to account for API changes when doing so. - project: 'freedesktop/ci-templates' ref: *template_sha file: '/templates/debian.yml' @@ -15,6 +57,8 @@ include: file: '/templates/ci-fairy.yml' +# Define the build stages. These are used for UI grouping as well as +# dependencies. stages: - review - container_prep @@ -22,6 +66,7 @@ stages: - pages +# Base variables used for anything using a Debian environment .debian: variables: FDO_DISTRIBUTION_VERSION: buster @@ -50,6 +95,9 @@ check-commit: junit: results.xml +# Build our base container image, which contains the core distribution, the +# toolchain, and all our build dependencies. This will be reused in the build +# stage. container_prep: extends: - .ci-rules @@ -59,6 +107,8 @@ container_prep: stage: container_prep +# Core templates for all of our build steps. These are reused by all build jobs +# through the `extends` keyword. .build-env: extends: - .ci-rules @@ -74,6 +124,9 @@ container_prep: - export TESTS_RES_PATH="$BUILDDIR/tests-res.txt" - mkdir "$BUILDDIR" "$PREFIX" +# Extends the core build templates to also provide for running our testing. We +# run this inside a virtme (qemu wrapper) VM environment so we can test the DRM +# backend using the 'vkms' virtual driver under Linux. .build-and-test: extends: .build-env tags: @@ -101,6 +154,7 @@ container_prep: reports: junit: $BUILDDIR/meson-logs/testlog.junit.xml +# Same as above, but without running any tests. .build-no-test: extends: .build-env tags: @@ -154,6 +208,13 @@ build-no-gl: -Dlauncher-libseat=true extends: .build-and-test +# Expose docs and coverage reports, so we can show users any changes to these +# inside their merge requests, letting us check them before merge. +# +# This does not build the docs or coverage information itself, but just reuses +# the docs and coverage information from the x86-64 Debian builds as the +# canonical sources of coverage information; the docs themselves should be +# invariant across any architecture or OS. docs-and-coverage: extends: - .ci-rules @@ -177,7 +238,11 @@ docs-and-coverage: - Documentation/ - Test_Coverage/ -# does not inherit .ci-rules +# Generate the documentation for https://wayland.pages.freedesktop.org/weston/ +# Anything under public/ is published to this URL. +# +# Does not inherit .ci-rules as it should only run in our default branch for +# the upstream repo. pages: extends: - .debian diff --git a/.gitlab-ci/debian-install.sh b/.gitlab-ci/debian-install.sh index d9bfa305..bcdb9e9a 100644 --- a/.gitlab-ci/debian-install.sh +++ b/.gitlab-ci/debian-install.sh @@ -1,4 +1,8 @@ #!/bin/bash +# +# Constructs the base container image used to build Weston within CI. Per the +# comment at the top of .gitlab-ci.yml, any changes in this file must bump the +# $FDO_DISTRIBUTION_TAG variable so we know the container has to be rebuilt. set -o xtrace -o errexit @@ -99,13 +103,23 @@ apt-get -y --no-install-recommends install \ apt-get -y --no-install-recommends -t buster-backports install \ freerdp2-dev +# Build and install Meson. Generally we want to keep this in sync with what +# we require inside meson.build, however per wayland/weston@bcf37c937a36, +# we use a higher version here pip3 install --user git+https://github.com/mesonbuild/meson.git@0.57.0 export PATH=$HOME/.local/bin:$PATH -# for documentation + +# Our docs are built using Sphinx (top-level organisation and final HTML/CSS +# generation), Doxygen (parse structures/functions/comments from source code), +# Breathe (a bridge between Doxygen and Sphinx), and we use the Read the Docs +# theme for the final presentation. pip3 install sphinx==2.1.0 --user pip3 install breathe==4.13.0.post0 --user pip3 install sphinx_rtd_theme==0.4.3 --user +# Build a Linux kernel for use in testing. We enable the VKMS module so we can +# predictably test the DRM backend in the absence of real hardware. We lock the +# version here so we see predictable results. apt-get -y --no-install-recommends install $LINUX_DEV_PKGS git clone --depth=1 --branch=drm-next-2020-06-11-1 https://anongit.freedesktop.org/git/drm/drm.git linux cd linux @@ -120,7 +134,11 @@ mv linux/arch/x86/boot/bzImage /weston-virtme/bzImage mv linux/.config /weston-virtme/.config rm -rf linux -# Link to upstream virtme: https://github.com/amluto/virtme +# Build virtme, a QEMU wrapper: https://github.com/amluto/virtme +# +# virtme makes our lives easier by abstracting handling of the console, +# filesystem, etc, so we can pretend that the VM we execute in is actually +# just a regular container. # # The reason why we are using a fork here is that it adds a patch to have the # --script-dir command line option. With that we can run scripts that are in a @@ -137,6 +155,8 @@ git checkout -b snapshot 69e3cb83b3405edc99fcf9611f50012a4f210f78 ./setup.py install cd .. +# Build and install Wayland; keep this version in sync with our dependency +# in meson.build. git clone --branch 1.18.0 --depth=1 https://gitlab.freedesktop.org/wayland/wayland cd wayland git show -s HEAD @@ -146,6 +166,9 @@ cd build make install cd ../../ +# Keep this version in sync with our dependency in meson.build. If you wish to +# raise a MR against custom protocol, please change this reference to clone +# your relevant tree, and make sure you bump $FDO_DISTRIBUTION_TAG. git clone --branch 1.19 https://gitlab.freedesktop.org/wayland/wayland-protocols cd wayland-protocols git show -s HEAD @@ -156,6 +179,13 @@ make install cd ../../ rm -rf wayland-protocols +# Build and install our own version of Mesa. Debian provides a perfectly usable +# Mesa, however llvmpipe's rendering behaviour can change subtly over time. +# This doesn't work for our tests which expect pixel-precise reproduction, so +# we lock it to a set version for more predictability. If you need newer +# features from Mesa then bump this version and $FDO_DISTRIBUTION_TAG, however +# please be prepared for some of the tests to change output, which will need to +# be manually inspected for correctness. apt-get -y --no-install-recommends install $MESA_DEV_PKGS git clone --single-branch --branch 20.3 --shallow-since='2020-12-15' https://gitlab.freedesktop.org/mesa/mesa.git mesa cd mesa @@ -166,6 +196,9 @@ ninja ${NINJAFLAGS} -C build install cd .. rm -rf mesa +# PipeWire is used for remoting support. Unlike our other dependencies its +# behaviour will be stable, however as a pre-1.0 project its API is not yet +# stable, so again we lock it to a fixed version. rm -rf pipewire git clone --depth=1 --branch 0.3.31 https://gitlab.freedesktop.org/pipewire/pipewire.git pipewire cd pipewire @@ -174,6 +207,9 @@ ninja ${NINJAFLAGS} -C build install cd .. rm -rf pipewire +# seatd lets us avoid the pain of handling VTs manually through weston-launch +# or open-coding TTY assignment within Weston. We use this for our tests using +# the DRM backend. git clone --depth=1 --branch 0.5.0 https://git.sr.ht/~kennylevinsen/seatd cd seatd meson build -Dauto_features=disabled \