CI: Document the build and container process

Explain what we do within our CI and why, with links as required.

Signed-off-by: Daniel Stone <daniels@collabora.com>

.gitlab-ci.yml

@@ -1,12 +1,54 @@
 # vim: set expandtab shiftwidth=2 tabstop=8 textwidth=0:
 
 .templates_sha: &template_sha 567700e483aabed992d0a4fea84994a0472deff6 # see https://docs.gitlab.com/ee/ci/yaml/#includefile
+# This file uses the freedesktop ci-templates to build Weston and run our
+# tests in CI.
+# 
+# ci-templates uses a multi-stage build process. First, the base container
+# image is built which contains the core distribution, the toolchain, and
+# all our build dependencies. This container is aggressively cached; if a
+# container image matching $FDO_DISTRIBUTION_TAG is found in either the
+# upstream repo (wayland/weston) or the user's downstream repo, it is
+# reused for the build. This gives us predictability of build and far
+# quicker runtimes, however it means that any changes to the base container
+# must also change $FDO_DISTRIBUTION_TAG. When changing this, please use
+# the current date as well as a unique build identifier.
+#
+# After the container is either rebuilt (tag mismatch) or reused (tag
+# previously used), the build stage executes within this container.
+#
+# The final stage is used to expose documentation and coverage information,
+# including publishing documentation to the public site when built on the
+# main branch.
+#
+# Apart from the 'variables', 'include', and 'stages' top-level anchors,
+# everything not beginning with a dot ('.') is the name of a job which will
+# be executed as part of CI, unless the rules specify that it should not be
+# run.
+#
+# Variables prefixed with CI_ are generally provided by GitLab itself;
+# variables prefixed with FDO_ and templates prefixed by .fdo are provided
+# by the ci-templates.
+#
+# For more information on GitLab CI, including the YAML syntax, see:
+#   https://docs.gitlab.com/ee/ci/yaml/README.html
+#
+# Note that freedesktop.org uses the 'Community Edition' of GitLab, so features
+# marked as 'premium' or 'ultimate' are not available to us.
+#
+# For more information on ci-templates, see:
+#   - documentation at https://freedesktop.pages.freedesktop.org/ci-templates/
+#   - repo at https://gitlab.freedesktop.org/freedesktop/ci-templates/
 
 variables:
   FDO_UPSTREAM_REPO: wayland/weston
 
 
 include:
+  # Here we use a fixed ref in order to isolate ourselves from ci-templates
+  # API changes. If you need new features from ci-templates you must bump
+  # this to the current SHA you require from the ci-templates repo, however
+  # be aware that you may need to account for API changes when doing so.
   - project: 'freedesktop/ci-templates'
     ref: *template_sha
     file: '/templates/debian.yml'
@@ -15,6 +57,8 @@ include:
     file: '/templates/ci-fairy.yml'
 
 
+# Define the build stages. These are used for UI grouping as well as
+# dependencies.
 stages:
   - review
   - container_prep
@@ -22,6 +66,7 @@ stages:
   - pages
 
 
+# Base variables used for anything using a Debian environment
 .debian:
   variables:
     FDO_DISTRIBUTION_VERSION: buster
@@ -50,6 +95,9 @@ check-commit:
       junit: results.xml
 
 
+# Build our base container image, which contains the core distribution, the
+# toolchain, and all our build dependencies. This will be reused in the build
+# stage.
 container_prep:
   extends:
     - .ci-rules
@@ -59,6 +107,8 @@ container_prep:
   stage: container_prep
 
 
+# Core templates for all of our build steps. These are reused by all build jobs
+# through the `extends` keyword.
 .build-env:
   extends:
     - .ci-rules
@@ -74,6 +124,9 @@ container_prep:
   - export TESTS_RES_PATH="$BUILDDIR/tests-res.txt"
   - mkdir "$BUILDDIR" "$PREFIX"
 
+# Extends the core build templates to also provide for running our testing. We
+# run this inside a virtme (qemu wrapper) VM environment so we can test the DRM
+# backend using the 'vkms' virtual driver under Linux.
 .build-and-test:
   extends: .build-env
   tags:
@@ -101,6 +154,7 @@ container_prep:
     reports:
       junit: $BUILDDIR/meson-logs/testlog.junit.xml
 
+# Same as above, but without running any tests.
 .build-no-test:
   extends: .build-env
   tags:
@@ -154,6 +208,13 @@ build-no-gl:
       -Dlauncher-libseat=true
   extends: .build-and-test
 
+# Expose docs and coverage reports, so we can show users any changes to these
+# inside their merge requests, letting us check them before merge.
+#
+# This does not build the docs or coverage information itself, but just reuses
+# the docs and coverage information from the x86-64 Debian builds as the
+# canonical sources of coverage information; the docs themselves should be
+# invariant across any architecture or OS.
 docs-and-coverage:
   extends:
     - .ci-rules
@@ -177,7 +238,11 @@ docs-and-coverage:
     - Documentation/
     - Test_Coverage/
 
-# does not inherit .ci-rules
+# Generate the documentation for https://wayland.pages.freedesktop.org/weston/
+# Anything under public/ is published to this URL.
+#
+# Does not inherit .ci-rules as it should only run in our default branch for
+# the upstream repo.
 pages:
   extends:
     - .debian

.gitlab-ci/debian-install.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+# 
+# Constructs the base container image used to build Weston within CI. Per the
+# comment at the top of .gitlab-ci.yml, any changes in this file must bump the
+# $FDO_DISTRIBUTION_TAG variable so we know the container has to be rebuilt.
 
 set -o xtrace -o errexit
 
@@ -99,13 +103,23 @@ apt-get -y --no-install-recommends install \
 apt-get -y --no-install-recommends -t buster-backports install \
 	freerdp2-dev
 
+# Build and install Meson. Generally we want to keep this in sync with what
+# we require inside meson.build, however per wayland/weston@bcf37c937a36,
+# we use a higher version here 
 pip3 install --user git+https://github.com/mesonbuild/meson.git@0.57.0
 export PATH=$HOME/.local/bin:$PATH
-# for documentation
+
+# Our docs are built using Sphinx (top-level organisation and final HTML/CSS
+# generation), Doxygen (parse structures/functions/comments from source code),
+# Breathe (a bridge between Doxygen and Sphinx), and we use the Read the Docs
+# theme for the final presentation.
 pip3 install sphinx==2.1.0 --user
 pip3 install breathe==4.13.0.post0 --user
 pip3 install sphinx_rtd_theme==0.4.3 --user
 
+# Build a Linux kernel for use in testing. We enable the VKMS module so we can
+# predictably test the DRM backend in the absence of real hardware. We lock the
+# version here so we see predictable results.
 apt-get -y --no-install-recommends install $LINUX_DEV_PKGS
 git clone --depth=1 --branch=drm-next-2020-06-11-1 https://anongit.freedesktop.org/git/drm/drm.git linux
 cd linux
@@ -120,7 +134,11 @@ mv linux/arch/x86/boot/bzImage /weston-virtme/bzImage
 mv linux/.config /weston-virtme/.config
 rm -rf linux
 
-# Link to upstream virtme: https://github.com/amluto/virtme
+# Build virtme, a QEMU wrapper: https://github.com/amluto/virtme
+# 
+# virtme makes our lives easier by abstracting handling of the console,
+# filesystem, etc, so we can pretend that the VM we execute in is actually
+# just a regular container.
 #
 # The reason why we are using a fork here is that it adds a patch to have the
 # --script-dir command line option. With that we can run scripts that are in a
@@ -137,6 +155,8 @@ git checkout -b snapshot 69e3cb83b3405edc99fcf9611f50012a4f210f78
 ./setup.py install
 cd ..
 
+# Build and install Wayland; keep this version in sync with our dependency
+# in meson.build.
 git clone --branch 1.18.0 --depth=1 https://gitlab.freedesktop.org/wayland/wayland
 cd wayland
 git show -s HEAD
@@ -146,6 +166,9 @@ cd build
 make install
 cd ../../
 
+# Keep this version in sync with our dependency in meson.build. If you wish to
+# raise a MR against custom protocol, please change this reference to clone
+# your relevant tree, and make sure you bump $FDO_DISTRIBUTION_TAG.
 git clone --branch 1.19 https://gitlab.freedesktop.org/wayland/wayland-protocols
 cd wayland-protocols
 git show -s HEAD
@@ -156,6 +179,13 @@ make install
 cd ../../
 rm -rf wayland-protocols
 
+# Build and install our own version of Mesa. Debian provides a perfectly usable
+# Mesa, however llvmpipe's rendering behaviour can change subtly over time.
+# This doesn't work for our tests which expect pixel-precise reproduction, so
+# we lock it to a set version for more predictability. If you need newer
+# features from Mesa then bump this version and $FDO_DISTRIBUTION_TAG, however
+# please be prepared for some of the tests to change output, which will need to
+# be manually inspected for correctness.
 apt-get -y --no-install-recommends install $MESA_DEV_PKGS
 git clone --single-branch --branch 20.3 --shallow-since='2020-12-15' https://gitlab.freedesktop.org/mesa/mesa.git mesa
 cd mesa
@@ -166,6 +196,9 @@ ninja ${NINJAFLAGS} -C build install
 cd ..
 rm -rf mesa
 
+# PipeWire is used for remoting support. Unlike our other dependencies its
+# behaviour will be stable, however as a pre-1.0 project its API is not yet
+# stable, so again we lock it to a fixed version.
 rm -rf pipewire
 git clone --depth=1 --branch 0.3.31 https://gitlab.freedesktop.org/pipewire/pipewire.git pipewire
 cd pipewire
@@ -174,6 +207,9 @@ ninja ${NINJAFLAGS} -C build install
 cd ..
 rm -rf pipewire
 
+# seatd lets us avoid the pain of handling VTs manually through weston-launch
+# or open-coding TTY assignment within Weston. We use this for our tests using
+# the DRM backend.
 git clone --depth=1 --branch 0.5.0 https://git.sr.ht/~kennylevinsen/seatd
 cd seatd
 meson build -Dauto_features=disabled \