Reduce CI go module downloads, add make targets (#18708)

The CI currently downloads all go modules in each pipeline step because go modules reside outside the project directory. Fix this by introducing a volume for the `/go` directory [1] so modules are only downloaded once per pipeline using a new `deps-backend` make target. For completeness, I also included new `deps` and `deps-frontend` targets and the frontend one is also triggered explicitly on CI where needed. [1] https://docs.drone.io/pipeline/kubernetes/examples/language/golang/#dependencies
3 years ago · 2ddff74a54
parent 4e57bd1d30
commit 2ddff74a54
3 changed files with 211 additions and 61 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -13,12 +13,25 @@ trigger:
    - tag
    - pull_request

+volumes:
+  - name: deps
+    temp: {}
+
 steps:
  - name: deps-frontend
-    pull: always
    image: node:16
+    pull: always
    commands:
-      - make node_modules
+      - make deps-frontend
+
+  - name: deps-backend
+    image: golang:1.17
+    pull: always
+    commands:
+      - make deps-backend
+    volumes:
+      - name: deps
+        path: /go

  - name: lint-frontend
    image: node:16
@ -27,17 +40,17 @@ steps:
    depends_on: [deps-frontend]

  - name: lint-backend
-    pull: always
    image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
+    pull: always
    commands:
      - make lint-backend
    environment:
      GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
      GOSUMDB: sum.golang.org
      TAGS: bindata sqlite sqlite_unlock_notify
+    depends_on: [deps-backend]

  - name: lint-backend-windows
-    pull: always
    image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
    commands:
      - make golangci-lint vet
@ -47,9 +60,9 @@ steps:
      TAGS: bindata sqlite sqlite_unlock_notify
      GOOS: windows
      GOARCH: amd64
+    depends_on: [deps-backend]

  - name: lint-backend-gogit
-    pull: always
    image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
    commands:
      - make lint-backend
@ -57,6 +70,7 @@ steps:
      GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
      GOSUMDB: sum.golang.org
      TAGS: bindata gogit sqlite sqlite_unlock_notify
+    depends_on: [deps-backend]

  - name: checks-frontend
    image: node:16
@ -65,11 +79,13 @@ steps:
    depends_on: [deps-frontend]

  - name: checks-backend
-    pull: always
    image: golang:1.17
    commands:
      - make checks-backend
-    depends_on: [lint-backend]
+    depends_on: [deps-backend]
+    volumes:
+      - name: deps
+        path: /go

  - name: test-frontend
    image: node:16
@ -84,14 +100,17 @@ steps:
    depends_on: [test-frontend]

  - name: build-backend-no-gcc
-    pull: always
    image: golang:1.16 # this step is kept as the lowest version of golang that we support
+    pull: always
    environment:
      GO111MODULE: on
      GOPROXY: https://goproxy.cn
    commands:
      - go build -o gitea_no_gcc # test if build succeeds without the sqlite tag
-    depends_on: [checks-backend]
+    depends_on: [deps-backend, checks-backend]
+    volumes:
+      - name: deps
+        path: /go

  - name: build-backend-arm64
    image: golang:1.17
@ -104,7 +123,10 @@ steps:
    commands:
      - make backend # test cross compile
      - rm ./gitea # clean
-    depends_on: [checks-backend]
+    depends_on: [deps-backend, checks-backend]
+    volumes:
+      - name: deps
+        path: /go

  - name: build-backend-windows
    image: golang:1.17
@ -116,7 +138,10 @@ steps:
      TAGS: bindata gogit
    commands:
      - go build -o gitea_windows
-    depends_on: [checks-backend]
+    depends_on: [deps-backend, checks-backend]
+    volumes:
+      - name: deps
+        path: /go

  - name: build-backend-386
    image: golang:1.17
@ -127,7 +152,10 @@ steps:
      GOARCH: 386
    commands:
      - go build -o gitea_linux_386 # test if compatible with 32 bit
-    depends_on: [checks-backend]
+    depends_on: [deps-backend, checks-backend]
+    volumes:
+      - name: deps
+        path: /go

 ---
 kind: pipeline
@ -147,21 +175,28 @@ trigger:
    - tag
    - pull_request

+volumes:
+  - name: deps
+    temp: {}
+
 services:
  - name: mysql
    image: mysql:5.7
+    pull: always
    environment:
      MYSQL_ALLOW_EMPTY_PASSWORD: yes
      MYSQL_DATABASE: test

  - name: mysql8
    image: mysql:8
+    pull: always
    environment:
      MYSQL_ALLOW_EMPTY_PASSWORD: yes
      MYSQL_DATABASE: testgitea

  - name: mssql
    image: mcr.microsoft.com/mssql/server:latest
+    pull: always
    environment:
      ACCEPT_EULA: Y
      MSSQL_PID: Standard
@ -169,14 +204,17 @@ services:

  - name: ldap
    image: gitea/test-openldap:latest
+    pull: always

  - name: elasticsearch
+    image: elasticsearch:7.5.0
+    pull: always
    environment:
      discovery.type: single-node
-    image: elasticsearch:7.5.0

  - name: minio
    image: minio/minio:RELEASE.2021-03-12T00-00-47Z
+    pull: always
    commands:
    - minio server /data
    environment:
@ -186,6 +224,7 @@ services:
 steps:
  - name: fetch-tags
    image: docker:git
+    pull: always
    commands:
      - git fetch --tags --force
    when:
@ -193,19 +232,28 @@ steps:
        exclude:
          - pull_request

-  - name: tag-pre-condition
+  - name: deps-backend
+    image: golang:1.17
    pull: always
+    commands:
+      - make deps-backend
+    volumes:
+      - name: deps
+        path: /go
+
+  - name: tag-pre-condition
    image: drone/git
+    pull: always
    commands:
      - git update-ref refs/heads/tag_test ${DRONE_COMMIT_SHA}

  - name: prepare-test-env
    image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
+    pull: always
    commands:
      - ./build/test-env-prepare.sh

  - name: build
-    pull: always
    image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
    user: gitea
    commands:
@ -215,8 +263,10 @@ steps:
      GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
      GOSUMDB: sum.golang.org
      TAGS: bindata sqlite sqlite_unlock_notify
-    depends_on:
-      - prepare-test-env
+    depends_on: [deps-backend, prepare-test-env]
+    volumes:
+      - name: deps
+        path: /go

  - name: unit-test
    image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
@ -229,9 +279,12 @@ steps:
      RACE_ENABLED: true
      GITHUB_READ_TOKEN:
        from_secret: github_read_token
+    depends_on: [deps-backend, prepare-test-env]
+    volumes:
+      - name: deps
+        path: /go

  - name: unit-test-gogit
-    pull: always
    image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
    user: gitea
    commands:
@ -242,6 +295,10 @@ steps:
      RACE_ENABLED: true
      GITHUB_READ_TOKEN:
        from_secret: github_read_token
+    depends_on: [deps-backend, prepare-test-env]
+    volumes:
+      - name: deps
+        path: /go

  - name: test-mysql
    image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
@ -255,8 +312,10 @@ steps:
      TEST_LDAP: 1
      USE_REPO_TEST_DIR: 1
      TEST_INDEXER_CODE_ES_URL: "http://elastic:changeme@elasticsearch:9200"
-    depends_on:
-      - build
+    depends_on: [build]
+    volumes:
+      - name: deps
+        path: /go

  - name: test-mysql8
    image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
@ -269,8 +328,10 @@ steps:
      RACE_ENABLED: true
      TEST_LDAP: 1
      USE_REPO_TEST_DIR: 1
-    depends_on:
-      - build
+    depends_on: [build]
+    volumes:
+      - name: deps
+        path: /go

  - name: test-mssql
    image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
@ -283,8 +344,10 @@ steps:
      RACE_ENABLED: true
      TEST_LDAP: 1
      USE_REPO_TEST_DIR: 1
-    depends_on:
-      - build
+    depends_on: [build]
+    volumes:
+      - name: deps
+        path: /go

  - name: generate-coverage
    image: golang:1.17
@ -293,9 +356,7 @@ steps:
    environment:
      GOPROXY: https://goproxy.cn
      TAGS: bindata
-    depends_on:
-      - unit-test
-      - test-mysql
+    depends_on: [unit-test, test-mysql]
    when:
      branch:
        - main
@ -304,15 +365,14 @@ steps:
        - pull_request

  - name: coverage-codecov
-    pull: always
    image: woodpeckerci/plugin-codecov:next-alpine
+    pull: always
    settings:
      files:
        - coverage.all
      token:
        from_secret: codecov_token
-    depends_on:
-      - generate-coverage
+    depends_on: [generate-coverage]
    when:
      branch:
        - main
@ -337,6 +397,10 @@ trigger:
    - tag
    - pull_request

+volumes:
+  - name: deps
+    temp: {}
+
 services:
  - name: pgsql
    pull: default
@ -352,6 +416,7 @@ services:
 steps:
  - name: fetch-tags
    image: docker:git
+    pull: always
    commands:
      - git fetch --tags --force
    when:
@ -359,13 +424,22 @@ steps:
        exclude:
          - pull_request

+  - name: deps-backend
+    image: golang:1.17
+    pull: always
+    commands:
+      - make deps-backend
+    volumes:
+      - name: deps
+        path: /go
+
  - name: prepare-test-env
    image: gitea/test_env:linux-arm64  # https://gitea.com/gitea/test-env
+    pull: always
    commands:
      - ./build/test-env-prepare.sh

  - name: build
-    pull: always
    image: gitea/test_env:linux-arm64  # https://gitea.com/gitea/test-env
    user: gitea
    commands:
@ -375,8 +449,10 @@ steps:
      GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
      GOSUMDB: sum.golang.org
      TAGS: bindata gogit sqlite sqlite_unlock_notify
-    depends_on:
-      - prepare-test-env
+    depends_on: [deps-backend, prepare-test-env]
+    volumes:
+      - name: deps
+        path: /go

  - name: test-sqlite
    image: gitea/test_env:linux-arm64  # https://gitea.com/gitea/test-env
@ -389,8 +465,10 @@ steps:
      RACE_ENABLED: true
      TEST_TAGS: gogit sqlite sqlite_unlock_notify
      USE_REPO_TEST_DIR: 1
-    depends_on:
-      - build
+    depends_on: [build]
+    volumes:
+      - name: deps
+        path: /go

  - name: test-pgsql
    image: gitea/test_env:linux-arm64  # https://gitea.com/gitea/test-env
@ -404,8 +482,10 @@ steps:
      TEST_TAGS: gogit
      TEST_LDAP: 1
      USE_REPO_TEST_DIR: 1
-    depends_on:
-      - build
+    depends_on: [build]
+    volumes:
+      - name: deps
+        path: /go

 ---
 kind: pipeline
@ -425,8 +505,8 @@ trigger:

 steps:
  - name: download
-    pull: always
    image: jonasfranz/crowdin
+    pull: always
    settings:
      download: true
      export_dir: options/locale/
@ -437,14 +517,14 @@ steps:
        from_secret: crowdin_key

  - name: update
-    pull: default
    image: alpine:3.13
+    pull: always
    commands:
      - ./build/update-locales.sh

  - name: push
-    pull: always
    image: appleboy/drone-git-push
+    pull: always
    settings:
      author_email: "teabot@gitea.io"
      author_name: GiteaBot
@ -457,8 +537,8 @@ steps:
        from_secret: git_push_ssh_key

  - name: upload_translations
-    pull: always
    image: jonasfranz/crowdin
+    pull: always
    settings:
      files:
        locale_en-US.ini: options/locale/locale_en-US.ini
@ -488,12 +568,13 @@ trigger:
 steps:
  - name: download
    image: golang:1.17
+    pull: always
    commands:
      - timeout -s ABRT 40m make generate-license generate-gitignore

  - name: push
-    pull: always
    image: appleboy/drone-git-push
+    pull: always
    settings:
      author_email: "teabot@gitea.io"
      author_name: GiteaBot
@ -529,15 +610,35 @@ depends_on:
  - testing-amd64
  - testing-arm64

+volumes:
+  - name: deps
+    temp: {}
+
 steps:
  - name: fetch-tags
    image: docker:git
+    pull: always
    commands:
      - git fetch --tags --force

-  - name: static
+  - name: deps-frontend
+    image: node:16
+    pull: always
+    commands:
+      - make deps-frontend
+
+  - name: deps-backend
+    image: golang:1.17
    pull: always
+    commands:
+      - make deps-backend
+    volumes:
+      - name: deps
+        path: /go
+
+  - name: static
    image: techknowlogick/xgo:go-1.17.x
+    pull: always
    commands:
      - curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt-get install -y nodejs
      - export PATH=$PATH:$GOPATH/bin
@ -545,10 +646,13 @@ steps:
    environment:
      GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
      TAGS: bindata sqlite sqlite_unlock_notify
+    volumes:
+      - name: deps
+        path: /go

  - name: gpg-sign
-    pull: always
    image: plugins/gpgsign:1
+    pull: always
    settings:
      detach_sign: true
      excludes:
@ -562,8 +666,8 @@ steps:
        from_secret: gpgsign_passphrase

  - name: release-branch
-    pull: always
    image: woodpeckerci/plugin-s3:latest
+    pull: always
    settings:
      acl: public-read
      bucket: gitea-artifacts
@ -624,16 +728,35 @@ depends_on:
  - testing-arm64
  - testing-amd64

+volumes:
+  - name: deps
+    temp: {}
+
 steps:
  - name: fetch-tags
-    pull: default
    image: docker:git
+    pull: always
    commands:
      - git fetch --tags --force

-  - name: static
+  - name: deps-frontend
+    image: node:16
+    pull: always
+    commands:
+      - make deps-frontend
+
+  - name: deps-backend
+    image: golang:1.17
    pull: always
+    commands:
+      - make deps-backend
+    volumes:
+      - name: deps
+        path: /go
+
+  - name: static
    image: techknowlogick/xgo:go-1.17.x
+    pull: always
    commands:
      - curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt-get install -y nodejs
      - export PATH=$PATH:$GOPATH/bin
@ -642,10 +765,13 @@ steps:
      GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
      TAGS: bindata sqlite sqlite_unlock_notify
    depends_on: [fetch-tags]
+    volumes:
+      - name: deps
+        path: /go

  - name: gpg-sign
-    pull: always
    image: plugins/gpgsign:1
+    pull: always
    settings:
      detach_sign: true
      excludes:
@ -660,8 +786,8 @@ steps:
    depends_on: [static]

  - name: release-tag
-    pull: always
    image: woodpeckerci/plugin-s3:latest
+    pull: always
    settings:
      acl: public-read
      bucket: gitea-artifacts
@ -678,8 +804,8 @@ steps:
    depends_on: [gpg-sign]

  - name: github
-    pull: always
    image: plugins/github-release:1
+    pull: always
    settings:
      files:
        - "dist/release/*"
@ -708,16 +834,16 @@ trigger:

 steps:
  - name: build-docs
-    pull: always
    image: plugins/hugo:latest
+    pull: always
    commands:
      - apk add --no-cache make bash curl
      - cd docs
      - make trans-copy clean build

  - name: publish-docs
-    pull: always
    image: techknowlogick/drone-netlify:latest
+    pull: always
    settings:
      path: docs/public/
      site_id: d2260bae-7861-4c02-8646-8f6440b12672
@ -753,12 +879,13 @@ trigger:
 steps:
  - name: fetch-tags
    image: docker:git
+    pull: always
    commands:
      - git fetch --tags --force

  - name: publish
-    pull: always
    image: techknowlogick/drone-docker:latest
+    pull: always
    settings:
      auto_tag: true
      auto_tag_suffix: linux-amd64
@ -815,12 +942,13 @@ trigger:
 steps:
  - name: fetch-tags
    image: docker:git
+    pull: always
    commands:
      - git fetch --tags --force

  - name: publish
-    pull: always
    image: techknowlogick/drone-docker:latest
+    pull: always
    settings:
      auto_tag: false
      tags: dev-linux-amd64
@ -876,12 +1004,13 @@ trigger:
 steps:
  - name: fetch-tags
    image: docker:git
+    pull: always
    commands:
      - git fetch --tags --force

  - name: publish
-    pull: always
    image: techknowlogick/drone-docker:latest
+    pull: always
    settings:
      auto_tag: false
      tags: ${DRONE_BRANCH##release/v}-dev-linux-amd64
@ -933,8 +1062,8 @@ trigger:

 steps:
  - name: dryrun
-    pull: always
    image: techknowlogick/drone-docker:latest
+    pull: always
    settings:
      dry_run: true
      repo: gitea/gitea
@ -971,12 +1100,13 @@ trigger:
 steps:
  - name: fetch-tags
    image: docker:git
+    pull: always
    commands:
      - git fetch --tags --force

  - name: publish
-    pull: always
    image: techknowlogick/drone-docker:latest
+    pull: always
    settings:
      auto_tag: true
      auto_tag_suffix: linux-arm64
@ -1033,12 +1163,13 @@ trigger:
 steps:
  - name: fetch-tags
    image: docker:git
+    pull: always
    commands:
      - git fetch --tags --force

  - name: publish
-    pull: always
    image: techknowlogick/drone-docker:latest
+    pull: always
    settings:
      auto_tag: false
      tags: dev-linux-arm64
@ -1094,12 +1225,13 @@ trigger:
 steps:
  - name: fetch-tags
    image: docker:git
+    pull: always
    commands:
      - git fetch --tags --force

  - name: publish
-    pull: always
    image: techknowlogick/drone-docker:latest
+    pull: always
    settings:
      auto_tag: false
      tags: ${DRONE_BRANCH##release/v}-dev-linux-arm64
@ -1144,8 +1276,8 @@ platform:

 steps:
  - name: manifest-rootless
-    pull: always
    image: plugins/manifest
+    pull: always
    settings:
      auto_tag: true
      ignore_missing: true
@ -1190,6 +1322,7 @@ steps:
  - name: manifest-rootless
    pull: always
    image: plugins/manifest
+    pull: always
    settings:
      auto_tag: false
      ignore_missing: true
@ -1264,8 +1397,8 @@ depends_on:

 steps:
  - name: discord
-    pull: always
    image: appleboy/drone-discord:1.2.4
+    pull: always
    settings:
      message: "{{#success build.status}} ✅  Build #{{build.number}} of `{{repo.name}}` succeeded.\n\n📝 Commit by {{commit.author}} on `{{commit.branch}}`:\n``` {{commit.message}} ```\n\n🌐 {{ build.link }} {{else}} ❌  Build #{{build.number}} of `{{repo.name}}` failed.\n\n📝 Commit by {{commit.author}} on `{{commit.branch}}`:\n``` {{commit.message}} ```\n\n🌐 {{ build.link }} {{/success}}\n"
      webhook_id:
--- a/.golangci.yml
+++ b/.golangci.yml
@ -24,6 +24,10 @@ linters:

 run:
  timeout: 3m
+  skip-dirs:
+    - node_modules
+    - public
+    - web_src

 linters-settings:
  gocritic:
--- a/13
+++ b/13
@ -166,6 +166,9 @@ help:
 	@echo " - watch-backend                    watch backend files and continuously rebuild"
 	@echo " - clean                            delete backend and integration files"
 	@echo " - clean-all                        delete backend, frontend and integration files"
+	@echo " - deps                             install dependencies"
+	@echo " - deps-frontend                    install frontend dependencies"
+	@echo " - deps-backend                     install backend dependencies"
 	@echo " - lint                             lint everything"
 	@echo " - lint-frontend                    lint frontend files"
 	@echo " - lint-backend                     lint backend files"
@ -662,6 +665,16 @@ docs:
 	fi
 	cd docs; make trans-copy clean build-offline;

+.PHONY: deps
+deps: deps-frontend deps-backend
+
+.PHONY: deps-frontend
+deps-frontend: node_modules
+
+.PHONY: deps-backend
+deps-backend:
+	$(GO) mod download
+
 node_modules: package-lock.json
 	npm install --no-save
 	@touch node_modules