Maintenance: Refactored GitLab CI configuration for better maintainability (phase 2).

This commit is contained in:
Martin Gruner
2022-06-17 15:26:11 +02:00
committed by Florian Liebe
parent 8297bd431c
commit d69a191737
42 changed files with 221 additions and 239 deletions

View File

@@ -1,3 +1,3 @@
include:
- '.gitlab/ci/includes/*.yml'
- '.gitlab/ci/__includes__/*.yml'
- '.gitlab/ci/*.yml'

View File

@@ -0,0 +1,5 @@
stages:
- pre
- lint
- test
- browser

View File

@@ -4,7 +4,6 @@ variables:
Z_LOCALES: "en-us:de-de"
# Browser tests and variables
RAILS_SERVE_STATIC_FILES: "true"
APP_RESTART_CMD: 'bundle exec rake environment zammad:ci:app:restart'
TZ: "Europe/London"
# docker-imap-devel variables required for browser tests
MAILNAME: "mail.test.dc.zammad.com"
@@ -20,4 +19,4 @@ variables:
# Temporary switch to enable the mobile front end for testing.
ENABLE_EXPERIMENTAL_MOBILE_FRONTEND: 'true'
# Compile assets only once.
CI_SKIP_ASSETS_PRECOMPILE: 'true'
CI_SKIP_ASSETS_PRECOMPILE: 'true'

View File

@@ -1,18 +1,5 @@
include:
# browser-core
- local: '/.gitlab/ci/browser-core/build.yml'
- local: '/.gitlab/ci/browser-core/autowizard_chrome.yml'
- local: '/.gitlab/ci/browser-core/autowizard_ff.yml'
- local: '/.gitlab/ci/browser-core/api_client_php.yml'
- local: '/.gitlab/ci/browser-core/api_client_ruby.yml'
- local: '/.gitlab/ci/browser-core/capybara_chrome.yml'
- local: '/.gitlab/ci/browser-core/capybara_ff.yml'
- local: '/.gitlab/ci/browser-core/chrome.yml'
- local: '/.gitlab/ci/browser-core/firefox.yml'
.template_browser-core: &template_browser-core
stage: browser-core
stage: browser
dependencies:
- browser:build
extends:
@@ -21,6 +8,7 @@ include:
- .tags_docker_low_concurrency
variables:
RAILS_ENV: "production"
APP_RESTART_CMD: 'bundle exec rake environment zammad:ci:app:restart'
script:
- script/build/test_slice_tests.sh $TEST_SLICE
- RAILS_ENV=test bundle exec rake db:create
@@ -40,7 +28,7 @@ include:
- .services_mysql_postgresql_elasticsearch_selenium_firefox_imap_redis_memcached
.template_browser-core_autowizard: &template_browser-core_autowizard
stage: browser-core
stage: browser
dependencies:
- browser:build
extends:
@@ -48,6 +36,7 @@ include:
- .tags_docker_low_concurrency
variables:
RAILS_ENV: "production"
APP_RESTART_CMD: 'bundle exec rake environment zammad:ci:app:restart'
AUTO_WIZARD_FILE: "contrib/auto_wizard_example.json"
TEST_FILE: "test/integration/auto_wizard_browser_test.rb"
script:
@@ -68,19 +57,8 @@ include:
- .template_browser-core_autowizard
- .services_postgresql_selenium_firefox
.template_browser-core_api_client: &template_browser-core_api_client
stage: browser-core
dependencies:
- browser:build
extends:
- .job_rules_default
- .services_mysql_postgresql_redis_memcached
- .tags_docker_low_concurrency
variables:
RAILS_ENV: "production"
.template_browser-core_capybara: &template_browser-core_capybara
stage: browser-core
stage: browser
dependencies:
- browser:build
extends:

View File

@@ -1,19 +0,0 @@
api_client_php:
extends:
- .template_browser-core_api_client
variables:
ZAMMAD_PHP_API_CLIENT_UNIT_TESTS_URL: "http://localhost:3000"
ZAMMAD_PHP_API_CLIENT_UNIT_TESTS_USERNAME: "admin@example.com"
ZAMMAD_PHP_API_CLIENT_UNIT_TESTS_PASSWORD: "test"
script:
- RAILS_ENV=test bundle exec rake db:create
- echo -e "\\e[0Ksection_start:`date +%s`:zammad_ci_test_start[collapsed=true]\\r\\e[0Kbundle exec rake zammad:ci:test:start"
- RAILS_ENV=test bundle exec rake zammad:ci:test:start zammad:setup:auto_wizard
- echo -e "\\e[0Ksection_end:`date +%s`:zammad_ci_test_start\\r\\e[0K"
- git clone https://github.com/zammad/zammad-api-client-php.git
- php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
- php composer-setup.php --install-dir=/usr/local/bin
- ln -s /usr/local/bin/composer.phar /usr/local/bin/composer
- cd zammad-api-client-php
- composer install
- vendor/bin/phpunit

View File

@@ -1,15 +0,0 @@
api_client_ruby:
extends:
- .template_browser-core_api_client
script:
- RAILS_ENV=test bundle exec rake db:create
- cp contrib/auto_wizard_test.json auto_wizard.json
- echo -e "\\e[0Ksection_start:`date +%s`:zammad_ci_test_start[collapsed=true]\\r\\e[0Kbundle exec rake zammad:ci:test:start"
- bundle exec rake zammad:ci:test:start
- echo -e "\\e[0Ksection_end:`date +%s`:zammad_ci_test_start\\r\\e[0K"
- git clone https://github.com/zammad/zammad-api-client-ruby.git
- cd zammad-api-client-ruby
- bundle update --bundler
- bundle config set --local frozen 'true'
- bundle install -j $(nproc)
- bundle exec rspec

View File

@@ -0,0 +1,7 @@
autowizard:chrome:
extends:
- .template_browser-core_autowizard_chrome
autowizard:firefox:
extends:
- .template_browser-core_autowizard_firefox

View File

@@ -1,3 +0,0 @@
autowizard_chrome:
extends:
- .template_browser-core_autowizard_chrome

View File

@@ -1,3 +0,0 @@
autowizard_ff:
extends:
- .template_browser-core_autowizard_firefox

View File

@@ -1,28 +1,28 @@
1_chrome:
minitest:chrome:1:
extends:
- .template_browser-core_chrome
variables:
TEST_SLICE: "1"
2_chrome:
minitest:chrome:2:
extends:
- .template_browser-core_chrome
variables:
TEST_SLICE: "2"
3_chrome:
minitest:chrome:3:
extends:
- .template_browser-core_chrome
variables:
TEST_SLICE: "3"
4_chrome:
minitest:chrome:4:
extends:
- .template_browser-core_chrome
variables:
TEST_SLICE: "4"
5_chrome:
minitest:chrome:5:
extends:
- .template_browser-core_chrome
variables:

View File

@@ -1,28 +1,28 @@
1_firefox:
minitest:firefox:1:
extends:
- .template_browser-core_firefox
variables:
TEST_SLICE: "1"
2_firefox:
minitest:firefox:2:
extends:
- .template_browser-core_firefox
variables:
TEST_SLICE: "2"
3_firefox:
minitest:firefox:3:
extends:
- .template_browser-core_firefox
variables:
TEST_SLICE: "3"
4_firefox:
minitest:firefox:4:
extends:
- .template_browser-core_firefox
variables:
TEST_SLICE: "4"
5_firefox:
minitest:firefox:5:
extends:
- .template_browser-core_firefox
variables:

View File

@@ -1,13 +1,5 @@
include:
# browser-integration
- local: '/.gitlab/ci/browser-integration/capybara_chrome.yml'
- local: '/.gitlab/ci/browser-integration/capybara_ff.yml'
- local: '/.gitlab/ci/browser-integration/idoit_chrome.yml'
- local: '/.gitlab/ci/browser-integration/otrs_chrome.yml'
.template_browser-integration_capybara: &template_browser-integration_capybara
stage: browser-integration
stage: browser
dependencies:
- browser:build
extends:
@@ -33,7 +25,7 @@ include:
- .services_mysql_postgresql_elasticsearch_selenium_firefox_imap_redis_memcached
.template_browser-integration_chrome: &template_browser-integration_chrome
stage: browser-integration
stage: browser
dependencies:
- browser:build
extends:
@@ -42,6 +34,7 @@ include:
- .tags_docker_low_concurrency
variables:
RAILS_ENV: "production"
APP_RESTART_CMD: 'bundle exec rake environment zammad:ci:app:restart'
AUTO_WIZARD_FILE: "contrib/auto_wizard_test.json"
script:
- RAILS_ENV=test bundle exec rake db:create
@@ -52,7 +45,7 @@ include:
- bundle exec rails test $TEST_FILE
.template_browser-integration_import_chrome: &template_browser-integration_import_chrome
stage: browser-integration
stage: browser
dependencies:
- browser:build
extends:
@@ -61,6 +54,7 @@ include:
- .tags_docker_low_concurrency
variables:
RAILS_ENV: "production"
APP_RESTART_CMD: 'bundle exec rake environment zammad:ci:app:restart'
RAILS_SERVE_STATIC_FILES: "true"
script:
- RAILS_ENV=test bundle exec rake db:create

View File

@@ -0,0 +1,7 @@
'integration:capybara:chrome':
extends:
- .template_browser-integration_capybara_chrome
'integration:capybara:firefox':
extends:
- .template_browser-integration_capybara_firefox

View File

@@ -1,3 +0,0 @@
integration_capybara_chrome:
extends:
- .template_browser-integration_capybara_chrome

View File

@@ -1,3 +0,0 @@
integration_capybara_ff:
extends:
- .template_browser-integration_capybara_firefox

View File

@@ -1,4 +1,4 @@
idoit_chrome:
'integration:minitest:idoit':
extends:
- .template_browser-integration_chrome
variables:

View File

@@ -1,4 +1,4 @@
otrs_chrome:
'integration:minitest:otrs':
extends:
- .template_browser-integration_import_chrome
variables:

3
.gitlab/ci/browser.yml Normal file
View File

@@ -0,0 +1,3 @@
include:
- '.gitlab/ci/browser-core/*.yml'
- '.gitlab/ci/browser-integration/*.yml'

View File

@@ -1,5 +0,0 @@
stages:
- pre
- test
- browser-core
- browser-integration

93
.gitlab/ci/lint.yml Normal file
View File

@@ -0,0 +1,93 @@
# Workaround to enable usage of mixed SSH and Docker GitLab CI runners
.template_lint: &template_lint
stage: lint
extends:
- .job_rules_default
before_script:
- source /etc/profile.d/rvm.sh # ensure RVM is loaded
.template_lint_rails: &template_lint_rails
stage: lint
extends:
- .job_rules_default
- .services_postgresql
before_script:
- source /etc/profile.d/rvm.sh # ensure RVM is loaded
- echo -e "\\e[0Ksection_start:`date +%s`:bundle_install[collapsed=true]\\r\\e[0Kbundle install"
- bundle config set --local frozen 'true'
- bundle config set --local path 'vendor'
- bundle install -j $(nproc)
- echo -e "\\e[0Ksection_end:`date +%s`:bundle_install\\r\\e[0K"
- bundle exec ruby .gitlab/configure_environment.rb
- source .gitlab/environment.env
- echo -e "\\e[0Ksection_start:`date +%s`:zammad_db_init[collapsed=true]\\r\\e[0Kbundle exec rake zammad:db:init"
- bundle exec rake zammad:db:init
- echo -e "\\e[0Ksection_end:`date +%s`:zammad_db_init\\r\\e[0K"
# Must be a separate job because it uses a custom image.
'lint: shellcheck':
<<: *template_lint
image: koalaman/shellcheck-alpine:stable
before_script:
- echo "Disable default before_script."
script:
- shellcheck -S warning $(find . -name "*.sh" -o -name "functions" | egrep -v "/vendor|node_modules/")
- shellcheck -S error script/init.d/*
after_script:
- echo "Disable default before_script."
'lint: i18n, rails & security':
<<: *template_lint_rails
artifacts:
expire_in: 1 week
paths:
- tmp/brakeman-report.html
when: on_failure
script:
- yarn install
- echo "Checking .po file syntax..."
- for FILE in i18n/*.pot i18n/*.po; do echo "Checking $FILE"; msgfmt -o /dev/null -c $FILE; done
- echo "Checking .pot catalog consistency..."
- bundle exec rails generate translation_catalog --check
- echo "Brakeman security check..."
- bundle exec brakeman -o /dev/stdout -o tmp/brakeman-report.html
- echo "bundler-audit security check..."
- gem install bundler-audit
- bundle-audit update
- bundle-audit --ignore CVE-2015-9284
- echo "Rails zeitwerk:check autoloader check..."
- bundle exec rails zeitwerk:check
- .gitlab/check_graphql_api_consistency.sh
'lint: ruby, js & css':
<<: *template_lint
script:
- echo "Coffeelint check..."
- coffeelint --rules ./.coffeelint/rules/* app/
- echo "Stylelint check..."
- yarn install
- yarn lint:css
- echo "ESLint check..."
- yarn install --cwd .storybook
- yarn lint
- echo "Storybook build..."
- yarn storybook:build
- echo "Rubocop check..."
- echo -e "\\e[0Ksection_start:`date +%s`:bundle_install[collapsed=true]\\r\\e[0Kbundle install"
- bundle config set --local frozen 'true'
- bundle config set --local path 'vendor'
- bundle install -j $(nproc)
- echo -e "\\e[0Ksection_end:`date +%s`:bundle_install\\r\\e[0K"
- bundle exec .rubocop/validate_todos.rb
- bundle exec rubocop --parallel
# Must be a separate job because it may fail and is only executed manually.
'lint: orphaned ruby gems':
<<: *template_lint_rails
allow_failure: true
rules:
- if: $CI_MERGE_REQUEST_ID
when: never
- when: manual
script:
- bundle exec rake zammad:ci:bundle:orphaned 5

View File

@@ -1,117 +1,22 @@
# Workaround to enable usage of mixed SSH and Docker GitLab CI runners
.template_pre: &template_pre
stage: pre
interruptible: true
extends:
- .job_rules_default
before_script:
- source /etc/profile.d/rvm.sh # ensure RVM is loaded
.template_pre_rails: &template_pre_rails
stage: pre
interruptible: true
extends:
- .job_rules_default
- .services_postgresql
before_script:
- source /etc/profile.d/rvm.sh # ensure RVM is loaded
- echo -e "\\e[0Ksection_start:`date +%s`:bundle_install[collapsed=true]\\r\\e[0Kbundle install"
- bundle config set --local frozen 'true'
- bundle config set --local path 'vendor'
- bundle install -j $(nproc)
- echo -e "\\e[0Ksection_end:`date +%s`:bundle_install\\r\\e[0K"
- bundle exec ruby .gitlab/configure_environment.rb
- source .gitlab/environment.env
- echo -e "\\e[0Ksection_start:`date +%s`:zammad_db_init[collapsed=true]\\r\\e[0Kbundle exec rake zammad:db:init"
- bundle exec rake zammad:db:init
- echo -e "\\e[0Ksection_end:`date +%s`:zammad_db_init\\r\\e[0K"
# Write CI cache once for faster runs
# Do not use anchor, it's gonna miss at least the relevant tag required
write_runner_cache:
'write runner cache':
<<: *template_pre
cache: !reference [.cache, read_write]
extends:
- .template_pre
script:
- bundle config set --local frozen 'true'
- bundle config set --local path 'vendor'
- bundle install -j $(nproc)
- yarn install
# Must be a separate job because it uses a custom image.
'lint: shellcheck':
<<: *template_pre
image: koalaman/shellcheck-alpine:stable
before_script:
- echo "Disable default before_script."
script:
- shellcheck -S warning $(find . -name "*.sh" -o -name "functions" | egrep -v "/vendor|node_modules/")
- shellcheck -S error script/init.d/*
after_script:
- echo "Disable default before_script."
'lint: i18n, rails & security':
<<: *template_pre_rails
needs:
- write_runner_cache
artifacts:
expire_in: 1 week
paths:
- tmp/brakeman-report.html
when: on_failure
script:
- yarn install
- echo "Checking .po file syntax..."
- for FILE in i18n/*.pot i18n/*.po; do echo "Checking $FILE"; msgfmt -o /dev/null -c $FILE; done
- echo "Checking .pot catalog consistency..."
- bundle exec rails generate translation_catalog --check
- echo "Brakeman security check..."
- bundle exec brakeman -o /dev/stdout -o tmp/brakeman-report.html
- echo "bundler-audit security check..."
- gem install bundler-audit
- bundle-audit update
- bundle-audit --ignore CVE-2015-9284
- echo "Rails zeitwerk:check autoloader check..."
- bundle exec rails zeitwerk:check
- .gitlab/check_graphql_api_consistency.sh
'lint: ruby, js & css':
<<: *template_pre
needs:
- write_runner_cache
script:
- echo "Coffeelint check..."
- coffeelint --rules ./.coffeelint/rules/* app/
- echo "Stylelint check..."
- yarn install
- yarn lint:css
- echo "ESLint check..."
- yarn install --cwd .storybook
- yarn lint
- echo "Storybook build..."
- yarn storybook:build
- echo "Rubocop check..."
- echo -e "\\e[0Ksection_start:`date +%s`:bundle_install[collapsed=true]\\r\\e[0Kbundle install"
- bundle config set --local frozen 'true'
- bundle config set --local path 'vendor'
- bundle install -j $(nproc)
- echo -e "\\e[0Ksection_end:`date +%s`:bundle_install\\r\\e[0K"
- bundle exec .rubocop/validate_todos.rb
- bundle exec rubocop --parallel
# Must be a separate job because it may fail and is only executed manually.
'lint: orphaned ruby gems':
<<: *template_pre_rails
allow_failure: true
needs:
- write_runner_cache
rules:
- if: $CI_MERGE_REQUEST_ID
when: never
- when: manual
script:
- bundle exec rake zammad:ci:bundle:orphaned 5
# Executed on a dedicated runner.
'push to github':
<<: *template_pre
@@ -125,13 +30,12 @@ write_runner_cache:
- git fetch --unshallow
- script/build/sync_repo.sh git@github.com:zammad/zammad.git
global_refresh_envs:
needs:
- write_runner_cache
'refresh global envs':
extends:
- .services_postgresql
stage: pre
interruptible: true
needs:
- "write runner cache"
# ensure that only one Job runs in the whole project (branches etc.)
resource_group: global_refresh_envs
# allow download via the web UI to restore ENVs in case global cache got deleted (see: `.gitlab-ci.yml`)
@@ -146,6 +50,5 @@ global_refresh_envs:
when: manual
allow_failure: true
- when: on_success
script:
- bundle exec rake zammad:ci:refresh_envs

2
.gitlab/ci/test.yml Normal file
View File

@@ -0,0 +1,2 @@
include:
- '.gitlab/ci/test/*.yml'

View File

@@ -0,0 +1,47 @@
.template_api_client:
stage: test
dependencies:
- browser:build
extends:
- .job_rules_default
- .services_mysql_postgresql_redis_memcached
- .tags_docker_low_concurrency
variables:
RAILS_ENV: "production"
APP_RESTART_CMD: 'bundle exec rake environment zammad:ci:app:restart'
api_client:php:
extends:
- .template_api_client
variables:
ZAMMAD_PHP_API_CLIENT_UNIT_TESTS_URL: "http://localhost:3000"
ZAMMAD_PHP_API_CLIENT_UNIT_TESTS_USERNAME: "admin@example.com"
ZAMMAD_PHP_API_CLIENT_UNIT_TESTS_PASSWORD: "test"
script:
- RAILS_ENV=test bundle exec rake db:create
- echo -e "\\e[0Ksection_start:`date +%s`:zammad_ci_test_start[collapsed=true]\\r\\e[0Kbundle exec rake zammad:ci:test:start"
- RAILS_ENV=test bundle exec rake zammad:ci:test:start zammad:setup:auto_wizard
- echo -e "\\e[0Ksection_end:`date +%s`:zammad_ci_test_start\\r\\e[0K"
- git clone https://github.com/zammad/zammad-api-client-php.git
- php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
- php composer-setup.php --install-dir=/usr/local/bin
- ln -s /usr/local/bin/composer.phar /usr/local/bin/composer
- cd zammad-api-client-php
- composer install
- vendor/bin/phpunit
api_client:ruby:
extends:
- .template_api_client
script:
- RAILS_ENV=test bundle exec rake db:create
- cp contrib/auto_wizard_test.json auto_wizard.json
- echo -e "\\e[0Ksection_start:`date +%s`:zammad_ci_test_start[collapsed=true]\\r\\e[0Kbundle exec rake zammad:ci:test:start"
- bundle exec rake zammad:ci:test:start
- echo -e "\\e[0Ksection_end:`date +%s`:zammad_ci_test_start\\r\\e[0K"
- git clone https://github.com/zammad/zammad-api-client-ruby.git
- cd zammad-api-client-ruby
- bundle update --bundler
- bundle config set --local frozen 'true'
- bundle install -j $(nproc)
- bundle exec rspec

View File

@@ -2,7 +2,6 @@ browser:build:
stage: test
extends:
- .services_postgresql
interruptible: true
artifacts:
expire_in: 1 week
paths:

View File

@@ -1,4 +1,17 @@
cypress:
frontend:vitest:
stage: test
extends:
- .job_rules_default
before_script:
- yarn install
script:
- yarn test
- echo "Also test eslint-plugin-zammad..."
- cd .eslint-plugin-zammad
- yarn install
- yarn test
frontend:cypress:
stage: test
extends:
- .job_rules_default

View File

@@ -1,7 +1,5 @@
include:
- local: '/.gitlab/ci/integration/es.yml'
- local: '/.gitlab/ci/integration/other.yml'
- local: '/.gitlab/ci/integration/otrs.yml'
- '.gitlab/ci/test/integration/*.yml'
.template_integration: &template_integration
extends:

View File

@@ -1,6 +1,5 @@
include:
- local: '/.gitlab/ci/rspec/mysql.yml'
- local: '/.gitlab/ci/rspec/postgresql.yml'
- '.gitlab/ci/test/rspec/*.yml'
.template_rspec: &template_rspec
extends:

View File

@@ -1,7 +1,3 @@
include:
- local: '/.gitlab/ci/unit/mysql.yml'
- local: '/.gitlab/ci/unit/postgresql.yml'
.template_unit: &template_unit
extends:
- .job_rules_default
@@ -14,3 +10,15 @@ include:
- bundle exec rake test:units
- bundle exec rails test test/integration/object_manager_test.rb
- bundle exec rails test test/integration/package_test.rb
unit:mysql:
stage: test
extends:
- .services_mysql_redis_memcached
- .template_unit
unit:postgresql:
stage: test
extends:
- .services_postgresql_redis_memcached
- .template_unit

View File

@@ -1,5 +0,0 @@
unit:mysql:
stage: test
extends:
- .services_mysql_redis_memcached
- .template_unit

View File

@@ -1,5 +0,0 @@
unit:postgresql:
stage: test
extends:
- .services_postgresql_redis_memcached
- .template_unit

View File

@@ -1,12 +0,0 @@
vitest:
stage: test
extends:
- .job_rules_default
before_script:
- yarn install
script:
- yarn test
- echo "Also test eslint-plugin-zammad..."
- cd .eslint-plugin-zammad
- yarn install
- yarn test