2021-02-22 17:28:32 +08:00
|
|
|
name: Tests
|
|
|
|
|
|
|
|
|
|
on:
|
|
|
|
|
pull_request:
|
2023-10-13 21:44:11 +08:00
|
|
|
paths-ignore:
|
|
|
|
|
- "migrations/**"
|
2021-02-22 17:28:32 +08:00
|
|
|
push:
|
|
|
|
|
branches:
|
|
|
|
|
- main
|
2022-03-18 10:15:48 +08:00
|
|
|
- beta
|
|
|
|
|
- stable
|
2023-10-13 21:44:11 +08:00
|
|
|
paths-ignore:
|
|
|
|
|
- "migrations/**"
|
2021-02-22 17:28:32 +08:00
|
|
|
|
2021-11-26 04:44:40 +08:00
|
|
|
concurrency:
|
2021-11-26 06:31:05 +08:00
|
|
|
group: tests-${{ format('{0}-{1}', github.head_ref || github.run_number, github.job) }}
|
2021-11-26 04:44:40 +08:00
|
|
|
cancel-in-progress: true
|
|
|
|
|
|
2022-07-30 23:22:03 +08:00
|
|
|
permissions:
|
|
|
|
|
contents: read
|
|
|
|
|
|
2021-02-22 17:28:32 +08:00
|
|
|
jobs:
|
|
|
|
|
build:
|
2023-05-12 20:00:04 +08:00
|
|
|
if: github.event_name == 'pull_request' || github.repository != 'discourse/discourse-private-mirror'
|
2024-01-22 19:10:17 +08:00
|
|
|
name: ${{ matrix.target }} ${{ matrix.build_type }}${{ !matrix.updated_ember && ' (Ember 3)' || '' }} # Update fetch-job-id step if changing this
|
2023-05-12 04:25:30 +08:00
|
|
|
runs-on: ${{ (matrix.build_type == 'annotations') && 'ubuntu-latest' || 'ubuntu-20.04-8core' }}
|
2023-02-23 02:39:49 +08:00
|
|
|
container: discourse/discourse_test:slim${{ (matrix.build_type == 'frontend' || matrix.build_type == 'system') && '-browsers' || '' }}${{ (matrix.ruby == '3.1') && '-ruby-3.1.0' || '' }}
|
2022-09-22 01:13:13 +08:00
|
|
|
timeout-minutes: 20
|
2021-02-22 17:28:32 +08:00
|
|
|
|
|
|
|
|
env:
|
|
|
|
|
DISCOURSE_HOSTNAME: www.example.com
|
|
|
|
|
RAILS_ENV: test
|
|
|
|
|
PGUSER: discourse
|
|
|
|
|
PGPASSWORD: discourse
|
2023-01-26 21:26:02 +08:00
|
|
|
USES_PARALLEL_DATABASES: ${{ matrix.build_type == 'backend' || matrix.build_type == 'system' }}
|
2023-05-29 10:41:24 +08:00
|
|
|
CAPYBARA_DEFAULT_MAX_WAIT_TIME: 10
|
2023-08-23 09:18:33 +08:00
|
|
|
MINIO_RUNNER_LOG_LEVEL: DEBUG
|
2023-12-06 20:12:04 +08:00
|
|
|
EMBER_VERSION: ${{ (matrix.updated_ember && '5') || '3' }}
|
2023-12-14 09:41:30 +08:00
|
|
|
DISCOURSE_TURBO_RSPEC_RETRY_AND_LOG_FLAKY_TESTS: ${{ (matrix.build_type == 'system' || matrix.build_type == 'backend') && github.ref_name == 'main' }}
|
2021-02-22 17:28:32 +08:00
|
|
|
|
|
|
|
|
strategy:
|
|
|
|
|
fail-fast: false
|
|
|
|
|
|
|
|
|
|
matrix:
|
DEV: Minimal first pass of rails system test setup (#16311)
This commit introduces rails system tests run with chromedriver, selenium,
and headless chrome to our testing toolbox.
We use the `webdrivers` gem and `selenium-webdriver` which is what
the latest Rails uses so the tests run locally and in CI out of the box.
You can use `SELENIUM_VERBOSE_DRIVER_LOGS=1` to show extra
verbose logs of what selenium is doing to communicate with the system
tests.
By default JS logs are verbose so errors from JS are shown when
running system tests, you can disable this with
`SELENIUM_DISABLE_VERBOSE_JS_LOGS=1`
You can use `SELENIUM_HEADLESS=0` to run the system
tests inside a chrome browser instead of headless, which can be useful to debug things
and see what the spec sees. See note above about `bin/ember-cli` to avoid
surprises.
I have modified `bin/turbo_rspec` to exclude `spec/system` by default,
support for parallel system specs is a little shaky right now and we don't
want them slowing down the turbo by default either.
### PageObjects and System Tests
To make querying and inspecting parts of the page easier
and more reusable inbetween system tests, we are using the
concept of [PageObjects](https://www.selenium.dev/documentation/test_practices/encouraged/page_object_models/) in
our system tests. A "Page" here is generally corresponds to
an overarching ember route, e.g. "Topic" for `/t/324345/some-topic`,
and this contains logic for querying components within the topic
such as "Posts".
I have also split "Modals" into their own entity. Further down the
line we may want to explore creating independent "Component"
contexts.
Capybara DSL should be included in each PageObject class,
reference for this can be found at https://rubydoc.info/github/teamcapybara/capybara/master#the-dsl
For system tests, since they are so slow, we want to focus on
the "happy path" and not do every different possible context
and branch check using them. They are meant to be overarching
tests that check a number of things are correct using the full stack
from JS and ember to rails to ruby and then the database.
### CI Setup
Whenever a system spec fails, a screenshot
is taken and a build artifact is produced _after the entire CI run is complete_,
which can be downloaded from the Actions UI in the repo.
Most importantly, a step to build the Ember app using Ember CLI
is needed, otherwise the JS assets cannot be found by capybara:
```
- name: Build Ember CLI
run: bin/ember-cli --build
```
A new `--build` argument has been added to `bin/ember-cli` for this
case, which is not needed locally if you already have the discourse
rails server running via `bin/ember-cli -u` since the whole server is built and
set up by default.
Co-authored-by: David Taylor <david@taylorhq.com>
2022-09-28 09:48:16 +08:00
|
|
|
build_type: [backend, frontend, system, annotations]
|
2023-11-16 07:11:35 +08:00
|
|
|
target: [core, plugins, themes]
|
2023-05-12 20:00:04 +08:00
|
|
|
ruby: ["3.2"]
|
2023-12-13 23:07:07 +08:00
|
|
|
updated_ember: [true]
|
2021-07-06 16:47:16 +08:00
|
|
|
exclude:
|
|
|
|
|
- build_type: annotations
|
|
|
|
|
target: plugins
|
2023-11-16 07:11:35 +08:00
|
|
|
- build_type: annotations
|
|
|
|
|
target: themes
|
|
|
|
|
- build_type: backend
|
|
|
|
|
target: themes
|
2022-01-19 18:41:52 +08:00
|
|
|
- build_type: frontend
|
|
|
|
|
target: core # Handled by core_frontend_tests job (below)
|
2023-12-15 22:33:04 +08:00
|
|
|
include:
|
|
|
|
|
- build_type: system
|
|
|
|
|
target: chat
|
|
|
|
|
updated_ember: true
|
2021-02-22 17:28:32 +08:00
|
|
|
|
|
|
|
|
steps:
|
2023-01-30 23:39:43 +08:00
|
|
|
- name: Set working directory owner
|
|
|
|
|
run: chown root:root .
|
|
|
|
|
|
2023-09-11 17:42:02 +08:00
|
|
|
- uses: actions/checkout@v4
|
2021-02-22 17:28:32 +08:00
|
|
|
with:
|
|
|
|
|
fetch-depth: 1
|
|
|
|
|
|
|
|
|
|
- name: Setup Git
|
|
|
|
|
run: |
|
|
|
|
|
git config --global user.email "ci@ci.invalid"
|
|
|
|
|
git config --global user.name "Discourse CI"
|
|
|
|
|
|
2021-09-08 20:01:37 +08:00
|
|
|
- name: Start redis
|
|
|
|
|
run: |
|
|
|
|
|
redis-server /etc/redis/redis.conf &
|
2021-02-22 17:28:32 +08:00
|
|
|
|
2021-12-15 01:20:06 +08:00
|
|
|
- name: Start Postgres
|
|
|
|
|
run: |
|
|
|
|
|
chown -R postgres /var/run/postgresql
|
|
|
|
|
sudo -E -u postgres script/start_test_db.rb
|
|
|
|
|
sudo -u postgres psql -c "CREATE ROLE $PGUSER LOGIN SUPERUSER PASSWORD '$PGPASSWORD';"
|
|
|
|
|
|
2021-02-22 17:28:32 +08:00
|
|
|
- name: Bundler cache
|
2024-01-22 18:50:56 +08:00
|
|
|
uses: actions/cache@v4
|
2021-02-22 17:28:32 +08:00
|
|
|
with:
|
|
|
|
|
path: vendor/bundle
|
2023-11-10 18:03:15 +08:00
|
|
|
key: ${{ runner.os }}-${{ matrix.ruby }}-gem-${{ hashFiles('**/Gemfile.lock') }}-cachev2
|
2021-02-22 17:28:32 +08:00
|
|
|
|
|
|
|
|
- name: Setup gems
|
|
|
|
|
run: |
|
2022-01-13 05:50:04 +08:00
|
|
|
gem install bundler --conservative -v $(awk '/BUNDLED WITH/ { getline; gsub(/ /,""); print $0 }' Gemfile.lock)
|
2021-02-22 17:28:32 +08:00
|
|
|
bundle config --local path vendor/bundle
|
|
|
|
|
bundle config --local deployment true
|
|
|
|
|
bundle config --local without development
|
|
|
|
|
bundle install --jobs 4
|
|
|
|
|
bundle clean
|
|
|
|
|
|
|
|
|
|
- name: Get yarn cache directory
|
|
|
|
|
id: yarn-cache-dir
|
2022-11-11 21:12:08 +08:00
|
|
|
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
|
2021-02-22 17:28:32 +08:00
|
|
|
|
|
|
|
|
- name: Yarn cache
|
2024-01-22 18:50:56 +08:00
|
|
|
uses: actions/cache@v4
|
2021-02-22 17:28:32 +08:00
|
|
|
id: yarn-cache
|
|
|
|
|
with:
|
|
|
|
|
path: ${{ steps.yarn-cache-dir.outputs.dir }}
|
2023-11-10 18:03:15 +08:00
|
|
|
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}-cachev2
|
2021-02-22 17:28:32 +08:00
|
|
|
|
|
|
|
|
- name: Yarn install
|
2023-05-02 23:25:22 +08:00
|
|
|
run: yarn install --frozen-lockfile
|
2021-02-22 17:28:32 +08:00
|
|
|
|
|
|
|
|
- name: Checkout official plugins
|
|
|
|
|
if: matrix.target == 'plugins'
|
|
|
|
|
run: bin/rake plugin:install_all_official
|
|
|
|
|
|
2022-03-18 10:49:58 +08:00
|
|
|
- name: Pull compatible versions of plugins
|
|
|
|
|
if: matrix.target == 'plugins'
|
|
|
|
|
run: bin/rake plugin:pull_compatible_all
|
|
|
|
|
|
2023-11-27 07:22:32 +08:00
|
|
|
- name: Plugin gems cache
|
2024-01-22 18:50:56 +08:00
|
|
|
uses: actions/cache@v4
|
2023-11-27 07:22:32 +08:00
|
|
|
with:
|
|
|
|
|
path: plugins/*/gems
|
|
|
|
|
key: ${{ runner.os }}-plugin-gems-${{matrix.ruby}}-${{ hashFiles('plugins/*/plugin.rb') }}
|
|
|
|
|
|
2023-11-16 07:11:35 +08:00
|
|
|
- name: Checkout official themes
|
2023-11-17 07:17:32 +08:00
|
|
|
if: matrix.target == 'themes' && matrix.build_type == 'system'
|
2023-11-16 07:11:35 +08:00
|
|
|
run: bin/rake themes:clone_all_official
|
|
|
|
|
|
2023-08-23 09:18:33 +08:00
|
|
|
- name: Add hosts to /etc/hosts, otherwise Chrome cannot reach minio
|
2023-12-07 11:46:14 +08:00
|
|
|
if: matrix.build_type == 'system' && matrix.target == 'core'
|
2023-08-23 09:18:33 +08:00
|
|
|
run: |
|
|
|
|
|
echo "127.0.0.1 minio.local" | sudo tee -a /etc/hosts
|
|
|
|
|
echo "127.0.0.1 discoursetest.minio.local" | sudo tee -a /etc/hosts
|
|
|
|
|
|
2021-12-14 17:40:16 +08:00
|
|
|
- name: Fetch app state cache
|
2024-01-22 18:50:56 +08:00
|
|
|
uses: actions/cache@v4
|
2021-12-14 17:40:16 +08:00
|
|
|
id: app-cache
|
|
|
|
|
with:
|
|
|
|
|
path: tmp/app-cache
|
2023-05-12 20:00:04 +08:00
|
|
|
key: >-
|
2021-12-14 17:40:16 +08:00
|
|
|
${{ runner.os }}-
|
|
|
|
|
${{ hashFiles('.github/workflows/tests.yml') }}-
|
|
|
|
|
${{ hashFiles('db/**/*', 'plugins/**/db/**/*') }}-
|
2024-01-29 09:57:58 +08:00
|
|
|
${{ hashFiles('config/environments/test.rb') }}-
|
2021-12-14 17:40:16 +08:00
|
|
|
${{ env.USES_PARALLEL_DATABASES }}
|
|
|
|
|
|
|
|
|
|
- name: Restore database from cache
|
|
|
|
|
if: steps.app-cache.outputs.cache-hit == 'true'
|
2023-11-16 23:55:41 +08:00
|
|
|
run: script/silence_successful_output psql --quiet -o /dev/null -f tmp/app-cache/cache.sql postgres
|
2021-12-14 17:40:16 +08:00
|
|
|
|
|
|
|
|
- name: Restore uploads from cache
|
|
|
|
|
if: steps.app-cache.outputs.cache-hit == 'true'
|
|
|
|
|
run: rm -rf public/uploads && cp -r tmp/app-cache/uploads public/uploads
|
|
|
|
|
|
|
|
|
|
- name: Create and migrate database
|
|
|
|
|
if: steps.app-cache.outputs.cache-hit != 'true'
|
2021-02-22 17:28:32 +08:00
|
|
|
run: |
|
|
|
|
|
bin/rake db:create
|
2023-11-16 23:55:41 +08:00
|
|
|
script/silence_successful_output bin/rake db:migrate
|
2021-02-22 17:28:32 +08:00
|
|
|
|
2021-12-14 17:40:16 +08:00
|
|
|
- name: Create and migrate parallel databases
|
|
|
|
|
if: >-
|
|
|
|
|
env.USES_PARALLEL_DATABASES == 'true' &&
|
|
|
|
|
steps.app-cache.outputs.cache-hit != 'true'
|
2021-02-22 17:28:32 +08:00
|
|
|
run: |
|
|
|
|
|
bin/rake parallel:create
|
2023-11-16 23:55:41 +08:00
|
|
|
script/silence_successful_output bin/rake parallel:migrate
|
2021-02-22 17:28:32 +08:00
|
|
|
|
2021-12-14 17:40:16 +08:00
|
|
|
- name: Dump database for cache
|
|
|
|
|
if: steps.app-cache.outputs.cache-hit != 'true'
|
|
|
|
|
run: mkdir -p tmp/app-cache && pg_dumpall > tmp/app-cache/cache.sql
|
|
|
|
|
|
|
|
|
|
- name: Dump uploads for cache
|
|
|
|
|
if: steps.app-cache.outputs.cache-hit != 'true'
|
|
|
|
|
run: rm -rf tmp/app-cache/uploads && cp -r public/uploads tmp/app-cache/uploads
|
|
|
|
|
|
2022-09-22 05:13:25 +08:00
|
|
|
- name: Fetch turbo_rspec_runtime.log cache
|
2024-01-22 18:50:56 +08:00
|
|
|
uses: actions/cache@v4
|
2022-09-22 05:13:25 +08:00
|
|
|
id: test-runtime-cache
|
2023-06-12 09:07:17 +08:00
|
|
|
if: matrix.build_type == 'backend' || matrix.build_type == 'system'
|
2022-09-22 05:13:25 +08:00
|
|
|
with:
|
|
|
|
|
path: tmp/turbo_rspec_runtime.log
|
2023-06-12 09:07:17 +08:00
|
|
|
key: rspec-runtime-${{ matrix.build_type }}-${{ matrix.target }}-${{ github.run_id }}
|
|
|
|
|
restore-keys: rspec-runtime-${{ matrix.build_type }}-${{ matrix.target }}-
|
2022-09-22 05:13:25 +08:00
|
|
|
|
2023-06-16 21:33:14 +08:00
|
|
|
- name: Check Zeitwerk reloading
|
|
|
|
|
if: matrix.build_type == 'backend'
|
|
|
|
|
env:
|
|
|
|
|
LOAD_PLUGINS: ${{ (matrix.target == 'plugins') && '1' || '0' }}
|
|
|
|
|
run: |
|
|
|
|
|
if ! bin/rails runner 'Rails.application.reloader.reload!'; then
|
|
|
|
|
echo
|
|
|
|
|
echo "---------------------------------------------"
|
|
|
|
|
echo
|
|
|
|
|
echo "::error::Zeitwerk reload failed - the app will not be able to reload properly in development."
|
|
|
|
|
echo "To reproduce locally, run \`bin/rails runner 'Rails.application.reloader.reload!'\`."
|
|
|
|
|
echo
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
|
2021-02-22 17:28:32 +08:00
|
|
|
- name: Core RSpec
|
|
|
|
|
if: matrix.build_type == 'backend' && matrix.target == 'core'
|
2023-12-14 09:41:30 +08:00
|
|
|
run: bin/turbo_rspec --use-runtime-info --verbose --format documentation
|
2021-02-22 17:28:32 +08:00
|
|
|
|
|
|
|
|
- name: Plugin RSpec
|
|
|
|
|
if: matrix.build_type == 'backend' && matrix.target == 'plugins'
|
2023-12-14 09:41:30 +08:00
|
|
|
run: bin/rake plugin:turbo_spec['*','--verbose --format documentation --use-runtime-info']
|
2021-02-22 17:28:32 +08:00
|
|
|
|
2022-06-20 22:33:05 +08:00
|
|
|
- name: Plugin QUnit
|
2022-07-17 04:55:39 +08:00
|
|
|
if: matrix.build_type == 'frontend' && matrix.target == 'plugins'
|
2023-10-10 23:29:28 +08:00
|
|
|
run: QUNIT_WRITE_EXECUTION_FILE=1 QUNIT_PARALLEL=3 bin/rake plugin:qunit['*','1200000']
|
2021-02-22 17:28:32 +08:00
|
|
|
timeout-minutes: 30
|
2021-07-06 16:47:16 +08:00
|
|
|
|
2023-11-17 07:17:32 +08:00
|
|
|
- name: Theme QUnit
|
|
|
|
|
if: matrix.build_type == 'frontend' && matrix.target == 'themes'
|
|
|
|
|
run: DISCOURSE_DEV_DB=discourse_test QUNIT_PARALLEL=3 bin/rake themes:qunit_all_official
|
|
|
|
|
timeout-minutes: 15
|
|
|
|
|
|
2023-12-18 18:47:22 +08:00
|
|
|
- uses: actions/upload-artifact@v4
|
2023-10-16 18:12:13 +08:00
|
|
|
if: always() && matrix.build_type == 'frontend' && matrix.target == 'plugins'
|
2023-10-10 23:29:28 +08:00
|
|
|
with:
|
2023-12-18 18:47:22 +08:00
|
|
|
name: ember-exam-execution-plugins-frontend-${{ hashFiles('./app/assets/javascripts/discourse/test-execution-*.json') }}
|
2023-10-10 23:29:28 +08:00
|
|
|
path: ./app/assets/javascripts/discourse/test-execution-*.json
|
|
|
|
|
|
DEV: Minimal first pass of rails system test setup (#16311)
This commit introduces rails system tests run with chromedriver, selenium,
and headless chrome to our testing toolbox.
We use the `webdrivers` gem and `selenium-webdriver` which is what
the latest Rails uses so the tests run locally and in CI out of the box.
You can use `SELENIUM_VERBOSE_DRIVER_LOGS=1` to show extra
verbose logs of what selenium is doing to communicate with the system
tests.
By default JS logs are verbose so errors from JS are shown when
running system tests, you can disable this with
`SELENIUM_DISABLE_VERBOSE_JS_LOGS=1`
You can use `SELENIUM_HEADLESS=0` to run the system
tests inside a chrome browser instead of headless, which can be useful to debug things
and see what the spec sees. See note above about `bin/ember-cli` to avoid
surprises.
I have modified `bin/turbo_rspec` to exclude `spec/system` by default,
support for parallel system specs is a little shaky right now and we don't
want them slowing down the turbo by default either.
### PageObjects and System Tests
To make querying and inspecting parts of the page easier
and more reusable inbetween system tests, we are using the
concept of [PageObjects](https://www.selenium.dev/documentation/test_practices/encouraged/page_object_models/) in
our system tests. A "Page" here is generally corresponds to
an overarching ember route, e.g. "Topic" for `/t/324345/some-topic`,
and this contains logic for querying components within the topic
such as "Posts".
I have also split "Modals" into their own entity. Further down the
line we may want to explore creating independent "Component"
contexts.
Capybara DSL should be included in each PageObject class,
reference for this can be found at https://rubydoc.info/github/teamcapybara/capybara/master#the-dsl
For system tests, since they are so slow, we want to focus on
the "happy path" and not do every different possible context
and branch check using them. They are meant to be overarching
tests that check a number of things are correct using the full stack
from JS and ember to rails to ruby and then the database.
### CI Setup
Whenever a system spec fails, a screenshot
is taken and a build artifact is produced _after the entire CI run is complete_,
which can be downloaded from the Actions UI in the repo.
Most importantly, a step to build the Ember app using Ember CLI
is needed, otherwise the JS assets cannot be found by capybara:
```
- name: Build Ember CLI
run: bin/ember-cli --build
```
A new `--build` argument has been added to `bin/ember-cli` for this
case, which is not needed locally if you already have the discourse
rails server running via `bin/ember-cli -u` since the whole server is built and
set up by default.
Co-authored-by: David Taylor <david@taylorhq.com>
2022-09-28 09:48:16 +08:00
|
|
|
- name: Ember Build for System Tests
|
|
|
|
|
if: matrix.build_type == 'system'
|
|
|
|
|
run: bin/ember-cli --build
|
|
|
|
|
|
2023-12-07 11:46:14 +08:00
|
|
|
- name: Ensure latest minio binary installed for Core System Tests
|
|
|
|
|
if: matrix.build_type == 'system' && matrix.target == 'core'
|
2023-10-24 10:43:14 +08:00
|
|
|
run: bundle exec ruby script/install_minio_binaries.rb
|
|
|
|
|
|
DEV: Minimal first pass of rails system test setup (#16311)
This commit introduces rails system tests run with chromedriver, selenium,
and headless chrome to our testing toolbox.
We use the `webdrivers` gem and `selenium-webdriver` which is what
the latest Rails uses so the tests run locally and in CI out of the box.
You can use `SELENIUM_VERBOSE_DRIVER_LOGS=1` to show extra
verbose logs of what selenium is doing to communicate with the system
tests.
By default JS logs are verbose so errors from JS are shown when
running system tests, you can disable this with
`SELENIUM_DISABLE_VERBOSE_JS_LOGS=1`
You can use `SELENIUM_HEADLESS=0` to run the system
tests inside a chrome browser instead of headless, which can be useful to debug things
and see what the spec sees. See note above about `bin/ember-cli` to avoid
surprises.
I have modified `bin/turbo_rspec` to exclude `spec/system` by default,
support for parallel system specs is a little shaky right now and we don't
want them slowing down the turbo by default either.
### PageObjects and System Tests
To make querying and inspecting parts of the page easier
and more reusable inbetween system tests, we are using the
concept of [PageObjects](https://www.selenium.dev/documentation/test_practices/encouraged/page_object_models/) in
our system tests. A "Page" here is generally corresponds to
an overarching ember route, e.g. "Topic" for `/t/324345/some-topic`,
and this contains logic for querying components within the topic
such as "Posts".
I have also split "Modals" into their own entity. Further down the
line we may want to explore creating independent "Component"
contexts.
Capybara DSL should be included in each PageObject class,
reference for this can be found at https://rubydoc.info/github/teamcapybara/capybara/master#the-dsl
For system tests, since they are so slow, we want to focus on
the "happy path" and not do every different possible context
and branch check using them. They are meant to be overarching
tests that check a number of things are correct using the full stack
from JS and ember to rails to ruby and then the database.
### CI Setup
Whenever a system spec fails, a screenshot
is taken and a build artifact is produced _after the entire CI run is complete_,
which can be downloaded from the Actions UI in the repo.
Most importantly, a step to build the Ember app using Ember CLI
is needed, otherwise the JS assets cannot be found by capybara:
```
- name: Build Ember CLI
run: bin/ember-cli --build
```
A new `--build` argument has been added to `bin/ember-cli` for this
case, which is not needed locally if you already have the discourse
rails server running via `bin/ember-cli -u` since the whole server is built and
set up by default.
Co-authored-by: David Taylor <david@taylorhq.com>
2022-09-28 09:48:16 +08:00
|
|
|
- name: Core System Tests
|
|
|
|
|
if: matrix.build_type == 'system' && matrix.target == 'core'
|
2023-12-14 09:41:30 +08:00
|
|
|
run: RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --use-runtime-info --profile=50 --verbose --format documentation spec/system
|
DEV: Minimal first pass of rails system test setup (#16311)
This commit introduces rails system tests run with chromedriver, selenium,
and headless chrome to our testing toolbox.
We use the `webdrivers` gem and `selenium-webdriver` which is what
the latest Rails uses so the tests run locally and in CI out of the box.
You can use `SELENIUM_VERBOSE_DRIVER_LOGS=1` to show extra
verbose logs of what selenium is doing to communicate with the system
tests.
By default JS logs are verbose so errors from JS are shown when
running system tests, you can disable this with
`SELENIUM_DISABLE_VERBOSE_JS_LOGS=1`
You can use `SELENIUM_HEADLESS=0` to run the system
tests inside a chrome browser instead of headless, which can be useful to debug things
and see what the spec sees. See note above about `bin/ember-cli` to avoid
surprises.
I have modified `bin/turbo_rspec` to exclude `spec/system` by default,
support for parallel system specs is a little shaky right now and we don't
want them slowing down the turbo by default either.
### PageObjects and System Tests
To make querying and inspecting parts of the page easier
and more reusable inbetween system tests, we are using the
concept of [PageObjects](https://www.selenium.dev/documentation/test_practices/encouraged/page_object_models/) in
our system tests. A "Page" here is generally corresponds to
an overarching ember route, e.g. "Topic" for `/t/324345/some-topic`,
and this contains logic for querying components within the topic
such as "Posts".
I have also split "Modals" into their own entity. Further down the
line we may want to explore creating independent "Component"
contexts.
Capybara DSL should be included in each PageObject class,
reference for this can be found at https://rubydoc.info/github/teamcapybara/capybara/master#the-dsl
For system tests, since they are so slow, we want to focus on
the "happy path" and not do every different possible context
and branch check using them. They are meant to be overarching
tests that check a number of things are correct using the full stack
from JS and ember to rails to ruby and then the database.
### CI Setup
Whenever a system spec fails, a screenshot
is taken and a build artifact is produced _after the entire CI run is complete_,
which can be downloaded from the Actions UI in the repo.
Most importantly, a step to build the Ember app using Ember CLI
is needed, otherwise the JS assets cannot be found by capybara:
```
- name: Build Ember CLI
run: bin/ember-cli --build
```
A new `--build` argument has been added to `bin/ember-cli` for this
case, which is not needed locally if you already have the discourse
rails server running via `bin/ember-cli -u` since the whole server is built and
set up by default.
Co-authored-by: David Taylor <david@taylorhq.com>
2022-09-28 09:48:16 +08:00
|
|
|
|
|
|
|
|
- name: Plugin System Tests
|
|
|
|
|
if: matrix.build_type == 'system' && matrix.target == 'plugins'
|
2023-10-25 19:58:36 +08:00
|
|
|
run: |
|
|
|
|
|
GLOBIGNORE="plugins/chat/*";
|
2024-01-17 11:00:32 +08:00
|
|
|
PREFABRICATION=0 LOAD_PLUGINS=1 RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --use-runtime-info --profile=50 --verbose --format documentation plugins/*/spec/system
|
2023-10-25 19:58:36 +08:00
|
|
|
shell: bash
|
|
|
|
|
timeout-minutes: 30
|
2023-11-16 07:11:35 +08:00
|
|
|
|
2023-10-25 19:58:36 +08:00
|
|
|
- name: Chat System Tests
|
|
|
|
|
if: matrix.build_type == 'system' && matrix.target == 'chat'
|
2023-12-14 09:41:30 +08:00
|
|
|
run: LOAD_PLUGINS=1 RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --use-runtime-info --profile=50 --verbose --format documentation plugins/chat/spec/system
|
2023-04-26 16:08:10 +08:00
|
|
|
timeout-minutes: 30
|
DEV: Minimal first pass of rails system test setup (#16311)
This commit introduces rails system tests run with chromedriver, selenium,
and headless chrome to our testing toolbox.
We use the `webdrivers` gem and `selenium-webdriver` which is what
the latest Rails uses so the tests run locally and in CI out of the box.
You can use `SELENIUM_VERBOSE_DRIVER_LOGS=1` to show extra
verbose logs of what selenium is doing to communicate with the system
tests.
By default JS logs are verbose so errors from JS are shown when
running system tests, you can disable this with
`SELENIUM_DISABLE_VERBOSE_JS_LOGS=1`
You can use `SELENIUM_HEADLESS=0` to run the system
tests inside a chrome browser instead of headless, which can be useful to debug things
and see what the spec sees. See note above about `bin/ember-cli` to avoid
surprises.
I have modified `bin/turbo_rspec` to exclude `spec/system` by default,
support for parallel system specs is a little shaky right now and we don't
want them slowing down the turbo by default either.
### PageObjects and System Tests
To make querying and inspecting parts of the page easier
and more reusable inbetween system tests, we are using the
concept of [PageObjects](https://www.selenium.dev/documentation/test_practices/encouraged/page_object_models/) in
our system tests. A "Page" here is generally corresponds to
an overarching ember route, e.g. "Topic" for `/t/324345/some-topic`,
and this contains logic for querying components within the topic
such as "Posts".
I have also split "Modals" into their own entity. Further down the
line we may want to explore creating independent "Component"
contexts.
Capybara DSL should be included in each PageObject class,
reference for this can be found at https://rubydoc.info/github/teamcapybara/capybara/master#the-dsl
For system tests, since they are so slow, we want to focus on
the "happy path" and not do every different possible context
and branch check using them. They are meant to be overarching
tests that check a number of things are correct using the full stack
from JS and ember to rails to ruby and then the database.
### CI Setup
Whenever a system spec fails, a screenshot
is taken and a build artifact is produced _after the entire CI run is complete_,
which can be downloaded from the Actions UI in the repo.
Most importantly, a step to build the Ember app using Ember CLI
is needed, otherwise the JS assets cannot be found by capybara:
```
- name: Build Ember CLI
run: bin/ember-cli --build
```
A new `--build` argument has been added to `bin/ember-cli` for this
case, which is not needed locally if you already have the discourse
rails server running via `bin/ember-cli -u` since the whole server is built and
set up by default.
Co-authored-by: David Taylor <david@taylorhq.com>
2022-09-28 09:48:16 +08:00
|
|
|
|
2023-11-16 07:11:35 +08:00
|
|
|
- name: Theme System Tests
|
|
|
|
|
if: matrix.build_type == 'system' && matrix.target == 'themes'
|
|
|
|
|
run: |
|
2023-12-14 09:41:30 +08:00
|
|
|
RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --profile=50 --verbose --format documentation tmp/themes/*/spec/system
|
2023-11-16 07:11:35 +08:00
|
|
|
shell: bash
|
|
|
|
|
timeout-minutes: 30
|
|
|
|
|
|
DEV: Introduce automatic reruns to RSpec tests on Github actions (#24811)
What motivated this change?
Our builds on Github actions have been extremely flaky mostly due to system tests. This has led to a drop in confidence
in our test suite where our developers tend to assume that a failed job is due to a flaky system test. As a result, we
have had occurrences where changes that resulted in legitimate test failures are merged into the `main` branch because developers
assumed it was a flaky test.
What does this change do?
This change seeks to reduce the flakiness of our builds on Github Actions by automatically re-running RSpec tests once when
they fail. If a failed test passes subsequently in the re-run, we mark the test as flaky by logging it into a file on disk
which is then uploaded as an artifact of the Github workflow run. We understand that automatically re-runs will lead to
lower accuracy of our tests but we accept this as an acceptable trade-off since a fragile build has a much greater impact
on our developers' time. Internally, the Discourse development team will be running a service to fetch the flaky tests
which have been logged for internal monitoring.
How is the change implemented?
1. A `--retry-and-log-flaky-tests` CLI flag is added to the `bin/turbo_rspec` CLI which will then initialize `TurboTests::Runner`
with the `retry_and_log_flaky_tests` kwarg set to `true`.
2. When the `retry_and_log_flaky_tests` kwarg is set to `true` for `TurboTests::Runner`, we will register an additional
formatter `Flaky::FailuresLoggerFormatter` to the `TurboTests::Reporter` in the `TurboTests::Runner#run` method.
The `Flaky::FailuresLoggerFormatter` has a simple job of logging all failed examples to a file on disk when running all the
tests. The details of the failed example which are logged can be found in `TurboTests::Flaky::FailedExample.to_h`.
3. Once all the tests have been run once, we check the result for any failed examples and if there are, we read the file on
disk to fetch the `location_rerun_location` of the failed examples which is then used to run the tests in a new RSpec process.
In the rerun, we configure a `TurboTests::Flaky::FlakyDetectorFormatter` with RSpec which removes all failed examples from the log file on disk since those examples are not flaky tests. Note that if there are too many failed examples on the first run, we will deem the failures to likely not be due to flaky tests and not re-run the test failures. As of writing, the threshold of failed examples is set to 10. If there are more than 10 failed examples, we will not re-run the failures.
2023-12-13 07:18:27 +08:00
|
|
|
- name: Check for failed system test screenshots
|
|
|
|
|
id: check-failed-system-test-screenshots
|
|
|
|
|
if: always() && matrix.build_type == 'system'
|
|
|
|
|
run: |
|
|
|
|
|
if [ -d tmp/capybara ] && [ "$(ls -A tmp/capybara/)" ]; then
|
|
|
|
|
echo "exists=true" >> $GITHUB_OUTPUT
|
|
|
|
|
else
|
|
|
|
|
echo "exists=false" >> $GITHUB_OUTPUT
|
|
|
|
|
fi
|
|
|
|
|
shell: bash
|
|
|
|
|
|
DEV: Minimal first pass of rails system test setup (#16311)
This commit introduces rails system tests run with chromedriver, selenium,
and headless chrome to our testing toolbox.
We use the `webdrivers` gem and `selenium-webdriver` which is what
the latest Rails uses so the tests run locally and in CI out of the box.
You can use `SELENIUM_VERBOSE_DRIVER_LOGS=1` to show extra
verbose logs of what selenium is doing to communicate with the system
tests.
By default JS logs are verbose so errors from JS are shown when
running system tests, you can disable this with
`SELENIUM_DISABLE_VERBOSE_JS_LOGS=1`
You can use `SELENIUM_HEADLESS=0` to run the system
tests inside a chrome browser instead of headless, which can be useful to debug things
and see what the spec sees. See note above about `bin/ember-cli` to avoid
surprises.
I have modified `bin/turbo_rspec` to exclude `spec/system` by default,
support for parallel system specs is a little shaky right now and we don't
want them slowing down the turbo by default either.
### PageObjects and System Tests
To make querying and inspecting parts of the page easier
and more reusable inbetween system tests, we are using the
concept of [PageObjects](https://www.selenium.dev/documentation/test_practices/encouraged/page_object_models/) in
our system tests. A "Page" here is generally corresponds to
an overarching ember route, e.g. "Topic" for `/t/324345/some-topic`,
and this contains logic for querying components within the topic
such as "Posts".
I have also split "Modals" into their own entity. Further down the
line we may want to explore creating independent "Component"
contexts.
Capybara DSL should be included in each PageObject class,
reference for this can be found at https://rubydoc.info/github/teamcapybara/capybara/master#the-dsl
For system tests, since they are so slow, we want to focus on
the "happy path" and not do every different possible context
and branch check using them. They are meant to be overarching
tests that check a number of things are correct using the full stack
from JS and ember to rails to ruby and then the database.
### CI Setup
Whenever a system spec fails, a screenshot
is taken and a build artifact is produced _after the entire CI run is complete_,
which can be downloaded from the Actions UI in the repo.
Most importantly, a step to build the Ember app using Ember CLI
is needed, otherwise the JS assets cannot be found by capybara:
```
- name: Build Ember CLI
run: bin/ember-cli --build
```
A new `--build` argument has been added to `bin/ember-cli` for this
case, which is not needed locally if you already have the discourse
rails server running via `bin/ember-cli -u` since the whole server is built and
set up by default.
Co-authored-by: David Taylor <david@taylorhq.com>
2022-09-28 09:48:16 +08:00
|
|
|
- name: Upload failed system test screenshots
|
2023-12-19 14:20:21 +08:00
|
|
|
uses: actions/upload-artifact@v3 # TODO (tgxworld): V4 doesn't allow us to upload multiple uploads to the same artifact name so keep at V3 for now.
|
DEV: Introduce automatic reruns to RSpec tests on Github actions (#24811)
What motivated this change?
Our builds on Github actions have been extremely flaky mostly due to system tests. This has led to a drop in confidence
in our test suite where our developers tend to assume that a failed job is due to a flaky system test. As a result, we
have had occurrences where changes that resulted in legitimate test failures are merged into the `main` branch because developers
assumed it was a flaky test.
What does this change do?
This change seeks to reduce the flakiness of our builds on Github Actions by automatically re-running RSpec tests once when
they fail. If a failed test passes subsequently in the re-run, we mark the test as flaky by logging it into a file on disk
which is then uploaded as an artifact of the Github workflow run. We understand that automatically re-runs will lead to
lower accuracy of our tests but we accept this as an acceptable trade-off since a fragile build has a much greater impact
on our developers' time. Internally, the Discourse development team will be running a service to fetch the flaky tests
which have been logged for internal monitoring.
How is the change implemented?
1. A `--retry-and-log-flaky-tests` CLI flag is added to the `bin/turbo_rspec` CLI which will then initialize `TurboTests::Runner`
with the `retry_and_log_flaky_tests` kwarg set to `true`.
2. When the `retry_and_log_flaky_tests` kwarg is set to `true` for `TurboTests::Runner`, we will register an additional
formatter `Flaky::FailuresLoggerFormatter` to the `TurboTests::Reporter` in the `TurboTests::Runner#run` method.
The `Flaky::FailuresLoggerFormatter` has a simple job of logging all failed examples to a file on disk when running all the
tests. The details of the failed example which are logged can be found in `TurboTests::Flaky::FailedExample.to_h`.
3. Once all the tests have been run once, we check the result for any failed examples and if there are, we read the file on
disk to fetch the `location_rerun_location` of the failed examples which is then used to run the tests in a new RSpec process.
In the rerun, we configure a `TurboTests::Flaky::FlakyDetectorFormatter` with RSpec which removes all failed examples from the log file on disk since those examples are not flaky tests. Note that if there are too many failed examples on the first run, we will deem the failures to likely not be due to flaky tests and not re-run the test failures. As of writing, the threshold of failed examples is set to 10. If there are more than 10 failed examples, we will not re-run the failures.
2023-12-13 07:18:27 +08:00
|
|
|
if: always() && steps.check-failed-system-test-screenshots.outputs.exists == 'true'
|
DEV: Minimal first pass of rails system test setup (#16311)
This commit introduces rails system tests run with chromedriver, selenium,
and headless chrome to our testing toolbox.
We use the `webdrivers` gem and `selenium-webdriver` which is what
the latest Rails uses so the tests run locally and in CI out of the box.
You can use `SELENIUM_VERBOSE_DRIVER_LOGS=1` to show extra
verbose logs of what selenium is doing to communicate with the system
tests.
By default JS logs are verbose so errors from JS are shown when
running system tests, you can disable this with
`SELENIUM_DISABLE_VERBOSE_JS_LOGS=1`
You can use `SELENIUM_HEADLESS=0` to run the system
tests inside a chrome browser instead of headless, which can be useful to debug things
and see what the spec sees. See note above about `bin/ember-cli` to avoid
surprises.
I have modified `bin/turbo_rspec` to exclude `spec/system` by default,
support for parallel system specs is a little shaky right now and we don't
want them slowing down the turbo by default either.
### PageObjects and System Tests
To make querying and inspecting parts of the page easier
and more reusable inbetween system tests, we are using the
concept of [PageObjects](https://www.selenium.dev/documentation/test_practices/encouraged/page_object_models/) in
our system tests. A "Page" here is generally corresponds to
an overarching ember route, e.g. "Topic" for `/t/324345/some-topic`,
and this contains logic for querying components within the topic
such as "Posts".
I have also split "Modals" into their own entity. Further down the
line we may want to explore creating independent "Component"
contexts.
Capybara DSL should be included in each PageObject class,
reference for this can be found at https://rubydoc.info/github/teamcapybara/capybara/master#the-dsl
For system tests, since they are so slow, we want to focus on
the "happy path" and not do every different possible context
and branch check using them. They are meant to be overarching
tests that check a number of things are correct using the full stack
from JS and ember to rails to ruby and then the database.
### CI Setup
Whenever a system spec fails, a screenshot
is taken and a build artifact is produced _after the entire CI run is complete_,
which can be downloaded from the Actions UI in the repo.
Most importantly, a step to build the Ember app using Ember CLI
is needed, otherwise the JS assets cannot be found by capybara:
```
- name: Build Ember CLI
run: bin/ember-cli --build
```
A new `--build` argument has been added to `bin/ember-cli` for this
case, which is not needed locally if you already have the discourse
rails server running via `bin/ember-cli -u` since the whole server is built and
set up by default.
Co-authored-by: David Taylor <david@taylorhq.com>
2022-09-28 09:48:16 +08:00
|
|
|
with:
|
2023-12-19 14:20:21 +08:00
|
|
|
name: failed-system-test-screenshots
|
2022-12-13 13:36:30 +08:00
|
|
|
path: tmp/capybara/*.png
|
DEV: Minimal first pass of rails system test setup (#16311)
This commit introduces rails system tests run with chromedriver, selenium,
and headless chrome to our testing toolbox.
We use the `webdrivers` gem and `selenium-webdriver` which is what
the latest Rails uses so the tests run locally and in CI out of the box.
You can use `SELENIUM_VERBOSE_DRIVER_LOGS=1` to show extra
verbose logs of what selenium is doing to communicate with the system
tests.
By default JS logs are verbose so errors from JS are shown when
running system tests, you can disable this with
`SELENIUM_DISABLE_VERBOSE_JS_LOGS=1`
You can use `SELENIUM_HEADLESS=0` to run the system
tests inside a chrome browser instead of headless, which can be useful to debug things
and see what the spec sees. See note above about `bin/ember-cli` to avoid
surprises.
I have modified `bin/turbo_rspec` to exclude `spec/system` by default,
support for parallel system specs is a little shaky right now and we don't
want them slowing down the turbo by default either.
### PageObjects and System Tests
To make querying and inspecting parts of the page easier
and more reusable inbetween system tests, we are using the
concept of [PageObjects](https://www.selenium.dev/documentation/test_practices/encouraged/page_object_models/) in
our system tests. A "Page" here is generally corresponds to
an overarching ember route, e.g. "Topic" for `/t/324345/some-topic`,
and this contains logic for querying components within the topic
such as "Posts".
I have also split "Modals" into their own entity. Further down the
line we may want to explore creating independent "Component"
contexts.
Capybara DSL should be included in each PageObject class,
reference for this can be found at https://rubydoc.info/github/teamcapybara/capybara/master#the-dsl
For system tests, since they are so slow, we want to focus on
the "happy path" and not do every different possible context
and branch check using them. They are meant to be overarching
tests that check a number of things are correct using the full stack
from JS and ember to rails to ruby and then the database.
### CI Setup
Whenever a system spec fails, a screenshot
is taken and a build artifact is produced _after the entire CI run is complete_,
which can be downloaded from the Actions UI in the repo.
Most importantly, a step to build the Ember app using Ember CLI
is needed, otherwise the JS assets cannot be found by capybara:
```
- name: Build Ember CLI
run: bin/ember-cli --build
```
A new `--build` argument has been added to `bin/ember-cli` for this
case, which is not needed locally if you already have the discourse
rails server running via `bin/ember-cli -u` since the whole server is built and
set up by default.
Co-authored-by: David Taylor <david@taylorhq.com>
2022-09-28 09:48:16 +08:00
|
|
|
|
DEV: Introduce automatic reruns to RSpec tests on Github actions (#24811)
What motivated this change?
Our builds on Github actions have been extremely flaky mostly due to system tests. This has led to a drop in confidence
in our test suite where our developers tend to assume that a failed job is due to a flaky system test. As a result, we
have had occurrences where changes that resulted in legitimate test failures are merged into the `main` branch because developers
assumed it was a flaky test.
What does this change do?
This change seeks to reduce the flakiness of our builds on Github Actions by automatically re-running RSpec tests once when
they fail. If a failed test passes subsequently in the re-run, we mark the test as flaky by logging it into a file on disk
which is then uploaded as an artifact of the Github workflow run. We understand that automatically re-runs will lead to
lower accuracy of our tests but we accept this as an acceptable trade-off since a fragile build has a much greater impact
on our developers' time. Internally, the Discourse development team will be running a service to fetch the flaky tests
which have been logged for internal monitoring.
How is the change implemented?
1. A `--retry-and-log-flaky-tests` CLI flag is added to the `bin/turbo_rspec` CLI which will then initialize `TurboTests::Runner`
with the `retry_and_log_flaky_tests` kwarg set to `true`.
2. When the `retry_and_log_flaky_tests` kwarg is set to `true` for `TurboTests::Runner`, we will register an additional
formatter `Flaky::FailuresLoggerFormatter` to the `TurboTests::Reporter` in the `TurboTests::Runner#run` method.
The `Flaky::FailuresLoggerFormatter` has a simple job of logging all failed examples to a file on disk when running all the
tests. The details of the failed example which are logged can be found in `TurboTests::Flaky::FailedExample.to_h`.
3. Once all the tests have been run once, we check the result for any failed examples and if there are, we read the file on
disk to fetch the `location_rerun_location` of the failed examples which is then used to run the tests in a new RSpec process.
In the rerun, we configure a `TurboTests::Flaky::FlakyDetectorFormatter` with RSpec which removes all failed examples from the log file on disk since those examples are not flaky tests. Note that if there are too many failed examples on the first run, we will deem the failures to likely not be due to flaky tests and not re-run the test failures. As of writing, the threshold of failed examples is set to 10. If there are more than 10 failed examples, we will not re-run the failures.
2023-12-13 07:18:27 +08:00
|
|
|
- name: Check for flaky tests report
|
|
|
|
|
id: check-flaky-spec-report
|
|
|
|
|
if: always()
|
|
|
|
|
run: |
|
|
|
|
|
if [ -f tmp/turbo_rspec_flaky_tests.json ]; then
|
|
|
|
|
echo "exists=true" >> $GITHUB_OUTPUT
|
|
|
|
|
else
|
|
|
|
|
echo "exists=false" >> $GITHUB_OUTPUT
|
|
|
|
|
fi
|
|
|
|
|
shell: bash
|
|
|
|
|
|
2023-12-18 15:59:41 +08:00
|
|
|
- name: Fetch Job ID
|
|
|
|
|
id: fetch-job-id
|
|
|
|
|
if: always() && steps.check-flaky-spec-report.outputs.exists == 'true'
|
|
|
|
|
run: |
|
2024-01-22 19:10:17 +08:00
|
|
|
job_id=$(ruby script/get_github_workflow_run_job_id.rb ${{ github.run_id }} ${{ github.run_attempt }} '${{ matrix.target }} ${{ matrix.build_type }}${{ !matrix.updated_ember && ' (Ember 3)' || '' }}')
|
2023-12-18 15:59:41 +08:00
|
|
|
echo "job_id=$job_id" >> $GITHUB_OUTPUT
|
|
|
|
|
|
DEV: Introduce automatic reruns to RSpec tests on Github actions (#24811)
What motivated this change?
Our builds on Github actions have been extremely flaky mostly due to system tests. This has led to a drop in confidence
in our test suite where our developers tend to assume that a failed job is due to a flaky system test. As a result, we
have had occurrences where changes that resulted in legitimate test failures are merged into the `main` branch because developers
assumed it was a flaky test.
What does this change do?
This change seeks to reduce the flakiness of our builds on Github Actions by automatically re-running RSpec tests once when
they fail. If a failed test passes subsequently in the re-run, we mark the test as flaky by logging it into a file on disk
which is then uploaded as an artifact of the Github workflow run. We understand that automatically re-runs will lead to
lower accuracy of our tests but we accept this as an acceptable trade-off since a fragile build has a much greater impact
on our developers' time. Internally, the Discourse development team will be running a service to fetch the flaky tests
which have been logged for internal monitoring.
How is the change implemented?
1. A `--retry-and-log-flaky-tests` CLI flag is added to the `bin/turbo_rspec` CLI which will then initialize `TurboTests::Runner`
with the `retry_and_log_flaky_tests` kwarg set to `true`.
2. When the `retry_and_log_flaky_tests` kwarg is set to `true` for `TurboTests::Runner`, we will register an additional
formatter `Flaky::FailuresLoggerFormatter` to the `TurboTests::Reporter` in the `TurboTests::Runner#run` method.
The `Flaky::FailuresLoggerFormatter` has a simple job of logging all failed examples to a file on disk when running all the
tests. The details of the failed example which are logged can be found in `TurboTests::Flaky::FailedExample.to_h`.
3. Once all the tests have been run once, we check the result for any failed examples and if there are, we read the file on
disk to fetch the `location_rerun_location` of the failed examples which is then used to run the tests in a new RSpec process.
In the rerun, we configure a `TurboTests::Flaky::FlakyDetectorFormatter` with RSpec which removes all failed examples from the log file on disk since those examples are not flaky tests. Note that if there are too many failed examples on the first run, we will deem the failures to likely not be due to flaky tests and not re-run the test failures. As of writing, the threshold of failed examples is set to 10. If there are more than 10 failed examples, we will not re-run the failures.
2023-12-13 07:18:27 +08:00
|
|
|
- name: Create flaky tests report artifact
|
|
|
|
|
if: always() && steps.check-flaky-spec-report.outputs.exists == 'true'
|
2023-12-18 15:59:41 +08:00
|
|
|
run: cp tmp/turbo_rspec_flaky_tests.json tmp/turbo_rspec_flaky_tests-${{ matrix.build_type }}-${{ matrix.target }}-${{ steps.fetch-job-id.outputs.job_id }}.json
|
DEV: Introduce automatic reruns to RSpec tests on Github actions (#24811)
What motivated this change?
Our builds on Github actions have been extremely flaky mostly due to system tests. This has led to a drop in confidence
in our test suite where our developers tend to assume that a failed job is due to a flaky system test. As a result, we
have had occurrences where changes that resulted in legitimate test failures are merged into the `main` branch because developers
assumed it was a flaky test.
What does this change do?
This change seeks to reduce the flakiness of our builds on Github Actions by automatically re-running RSpec tests once when
they fail. If a failed test passes subsequently in the re-run, we mark the test as flaky by logging it into a file on disk
which is then uploaded as an artifact of the Github workflow run. We understand that automatically re-runs will lead to
lower accuracy of our tests but we accept this as an acceptable trade-off since a fragile build has a much greater impact
on our developers' time. Internally, the Discourse development team will be running a service to fetch the flaky tests
which have been logged for internal monitoring.
How is the change implemented?
1. A `--retry-and-log-flaky-tests` CLI flag is added to the `bin/turbo_rspec` CLI which will then initialize `TurboTests::Runner`
with the `retry_and_log_flaky_tests` kwarg set to `true`.
2. When the `retry_and_log_flaky_tests` kwarg is set to `true` for `TurboTests::Runner`, we will register an additional
formatter `Flaky::FailuresLoggerFormatter` to the `TurboTests::Reporter` in the `TurboTests::Runner#run` method.
The `Flaky::FailuresLoggerFormatter` has a simple job of logging all failed examples to a file on disk when running all the
tests. The details of the failed example which are logged can be found in `TurboTests::Flaky::FailedExample.to_h`.
3. Once all the tests have been run once, we check the result for any failed examples and if there are, we read the file on
disk to fetch the `location_rerun_location` of the failed examples which is then used to run the tests in a new RSpec process.
In the rerun, we configure a `TurboTests::Flaky::FlakyDetectorFormatter` with RSpec which removes all failed examples from the log file on disk since those examples are not flaky tests. Note that if there are too many failed examples on the first run, we will deem the failures to likely not be due to flaky tests and not re-run the test failures. As of writing, the threshold of failed examples is set to 10. If there are more than 10 failed examples, we will not re-run the failures.
2023-12-13 07:18:27 +08:00
|
|
|
|
|
|
|
|
- name: Upload flaky tests report
|
2023-12-19 14:20:21 +08:00
|
|
|
uses: actions/upload-artifact@v3 # TODO (tgxworld): V4 doesn't allow us to upload multiple uploads to the same artifact name so keep at V3 for now
|
DEV: Introduce automatic reruns to RSpec tests on Github actions (#24811)
What motivated this change?
Our builds on Github actions have been extremely flaky mostly due to system tests. This has led to a drop in confidence
in our test suite where our developers tend to assume that a failed job is due to a flaky system test. As a result, we
have had occurrences where changes that resulted in legitimate test failures are merged into the `main` branch because developers
assumed it was a flaky test.
What does this change do?
This change seeks to reduce the flakiness of our builds on Github Actions by automatically re-running RSpec tests once when
they fail. If a failed test passes subsequently in the re-run, we mark the test as flaky by logging it into a file on disk
which is then uploaded as an artifact of the Github workflow run. We understand that automatically re-runs will lead to
lower accuracy of our tests but we accept this as an acceptable trade-off since a fragile build has a much greater impact
on our developers' time. Internally, the Discourse development team will be running a service to fetch the flaky tests
which have been logged for internal monitoring.
How is the change implemented?
1. A `--retry-and-log-flaky-tests` CLI flag is added to the `bin/turbo_rspec` CLI which will then initialize `TurboTests::Runner`
with the `retry_and_log_flaky_tests` kwarg set to `true`.
2. When the `retry_and_log_flaky_tests` kwarg is set to `true` for `TurboTests::Runner`, we will register an additional
formatter `Flaky::FailuresLoggerFormatter` to the `TurboTests::Reporter` in the `TurboTests::Runner#run` method.
The `Flaky::FailuresLoggerFormatter` has a simple job of logging all failed examples to a file on disk when running all the
tests. The details of the failed example which are logged can be found in `TurboTests::Flaky::FailedExample.to_h`.
3. Once all the tests have been run once, we check the result for any failed examples and if there are, we read the file on
disk to fetch the `location_rerun_location` of the failed examples which is then used to run the tests in a new RSpec process.
In the rerun, we configure a `TurboTests::Flaky::FlakyDetectorFormatter` with RSpec which removes all failed examples from the log file on disk since those examples are not flaky tests. Note that if there are too many failed examples on the first run, we will deem the failures to likely not be due to flaky tests and not re-run the test failures. As of writing, the threshold of failed examples is set to 10. If there are more than 10 failed examples, we will not re-run the failures.
2023-12-13 07:18:27 +08:00
|
|
|
if: always() && steps.check-flaky-spec-report.outputs.exists == 'true'
|
|
|
|
|
with:
|
2023-12-19 14:20:21 +08:00
|
|
|
name: flaky-test-reports
|
2023-12-18 15:59:41 +08:00
|
|
|
path: tmp/turbo_rspec_flaky_tests-${{ matrix.build_type }}-${{ matrix.target }}-${{ steps.fetch-job-id.outputs.job_id }}.json
|
DEV: Introduce automatic reruns to RSpec tests on Github actions (#24811)
What motivated this change?
Our builds on Github actions have been extremely flaky mostly due to system tests. This has led to a drop in confidence
in our test suite where our developers tend to assume that a failed job is due to a flaky system test. As a result, we
have had occurrences where changes that resulted in legitimate test failures are merged into the `main` branch because developers
assumed it was a flaky test.
What does this change do?
This change seeks to reduce the flakiness of our builds on Github Actions by automatically re-running RSpec tests once when
they fail. If a failed test passes subsequently in the re-run, we mark the test as flaky by logging it into a file on disk
which is then uploaded as an artifact of the Github workflow run. We understand that automatically re-runs will lead to
lower accuracy of our tests but we accept this as an acceptable trade-off since a fragile build has a much greater impact
on our developers' time. Internally, the Discourse development team will be running a service to fetch the flaky tests
which have been logged for internal monitoring.
How is the change implemented?
1. A `--retry-and-log-flaky-tests` CLI flag is added to the `bin/turbo_rspec` CLI which will then initialize `TurboTests::Runner`
with the `retry_and_log_flaky_tests` kwarg set to `true`.
2. When the `retry_and_log_flaky_tests` kwarg is set to `true` for `TurboTests::Runner`, we will register an additional
formatter `Flaky::FailuresLoggerFormatter` to the `TurboTests::Reporter` in the `TurboTests::Runner#run` method.
The `Flaky::FailuresLoggerFormatter` has a simple job of logging all failed examples to a file on disk when running all the
tests. The details of the failed example which are logged can be found in `TurboTests::Flaky::FailedExample.to_h`.
3. Once all the tests have been run once, we check the result for any failed examples and if there are, we read the file on
disk to fetch the `location_rerun_location` of the failed examples which is then used to run the tests in a new RSpec process.
In the rerun, we configure a `TurboTests::Flaky::FlakyDetectorFormatter` with RSpec which removes all failed examples from the log file on disk since those examples are not flaky tests. Note that if there are too many failed examples on the first run, we will deem the failures to likely not be due to flaky tests and not re-run the test failures. As of writing, the threshold of failed examples is set to 10. If there are more than 10 failed examples, we will not re-run the failures.
2023-12-13 07:18:27 +08:00
|
|
|
|
2021-07-06 16:47:16 +08:00
|
|
|
- name: Check Annotations
|
|
|
|
|
if: matrix.build_type == 'annotations'
|
|
|
|
|
run: |
|
|
|
|
|
bin/rake annotate:ensure_all_indexes
|
|
|
|
|
bin/annotate --models --model-dir app/models
|
|
|
|
|
|
|
|
|
|
if [ ! -z "$(git status --porcelain app/models/)" ]; then
|
|
|
|
|
echo "Core annotations are not up to date. To resolve, run:"
|
|
|
|
|
echo " bin/rake annotate:clean"
|
|
|
|
|
echo
|
|
|
|
|
echo "Or manually apply the diff printed below:"
|
|
|
|
|
echo "---------------------------------------------"
|
|
|
|
|
git -c color.ui=always diff app/models/
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
timeout-minutes: 30
|
2022-01-19 18:41:52 +08:00
|
|
|
|
|
|
|
|
core_frontend_tests:
|
2023-05-12 20:00:04 +08:00
|
|
|
if: github.event_name == 'pull_request' || github.repository != 'discourse/discourse-private-mirror'
|
2024-01-22 19:10:17 +08:00
|
|
|
name: core frontend (${{ matrix.browser }})${{ !matrix.updated_ember && ' (Ember 3)' || '' }}
|
2023-05-12 04:25:30 +08:00
|
|
|
runs-on: ubuntu-20.04-8core
|
2022-09-22 01:13:13 +08:00
|
|
|
container:
|
|
|
|
|
image: discourse/discourse_test:slim-browsers
|
|
|
|
|
options: --user discourse
|
|
|
|
|
|
2022-08-14 23:30:15 +08:00
|
|
|
timeout-minutes: 35
|
2022-01-19 18:41:52 +08:00
|
|
|
|
|
|
|
|
strategy:
|
|
|
|
|
fail-fast: false
|
|
|
|
|
matrix:
|
2022-09-21 22:34:26 +08:00
|
|
|
browser: ["Chrome", "Firefox ESR", "Firefox Evergreen"]
|
2023-12-13 23:07:07 +08:00
|
|
|
updated_ember: [true]
|
DEV: Introduce feature-flag for Ember 5 upgrade
This commit introduces the scaffolding for us to easily switch between Ember 3.28 and Ember 5 on the `main` branch of Discourse. Unfortunately, there is no built-in system to apply this kind of flagging within yarn / ember-cli. There are projects like `ember-try` which are designed for running against multiple version of a dependency, but they do not allow us to 'lock' dependency/sub-dependency versions, and are therefore unsuitable for our use in production.
Instead, we will be maintaining two root `package.json` files, and two `yarn.lock` files. For ember-3, they remain as-is. For ember5, we use a yarn 'resolution' to override the version for ember-source across the entire yarn workspace.
To allow for easy switching with minimal diff against the repository, `package.json` and `yarn.lock` are symlinks which point to `package-ember3.json` and `yarn-ember3.lock` by default. To switch to Ember 5, we can run `script/switch ember version 5` to update the symlinks to point to `package-ember5.json` and `package-ember3.json` respectively. In production, and when using `bin/ember-cli` for development, the ember version can also be upgraded using the `EMBER_VERSION=5` environment variable.
When making changes to dependencies, these should be made against the default `ember3` versions, and then `script/regen_ember_5_lockfile` should be used to regenerate `yarn-ember5.lock` accordingly. A new 'Ember Version Lockfiles' GitHub workflow will automate this process on Dependabot PRs.
When running a local environment against Ember 5, the two symlink changes will show up as git diffs. To avoid us accidentally committing/pushing that change, another GitHub workflow is introduced which checks the default Ember version and raises an error if it is greater than v3.
Supporting two ember versions simultaneously obviously carries significant overhead, so our aim will be to get themes/plugins updated as quickly as possible, and then drop this flag.
2023-11-27 23:52:17 +08:00
|
|
|
include:
|
|
|
|
|
- browser: Chrome
|
2023-12-13 23:07:07 +08:00
|
|
|
updated_ember: false
|
2022-09-21 22:34:26 +08:00
|
|
|
|
|
|
|
|
env:
|
|
|
|
|
TESTEM_BROWSER: ${{ (startsWith(matrix.browser, 'Firefox') && 'Firefox') || matrix.browser }}
|
|
|
|
|
TESTEM_FIREFOX_PATH: ${{ (matrix.browser == 'Firefox Evergreen') && '/opt/firefox-evergreen/firefox' }}
|
2022-01-19 18:41:52 +08:00
|
|
|
|
|
|
|
|
steps:
|
2023-09-11 17:42:02 +08:00
|
|
|
- uses: actions/checkout@v4
|
2022-01-19 18:41:52 +08:00
|
|
|
with:
|
|
|
|
|
fetch-depth: 1
|
|
|
|
|
|
|
|
|
|
- name: Setup Git
|
|
|
|
|
run: |
|
|
|
|
|
git config --global user.email "ci@ci.invalid"
|
|
|
|
|
git config --global user.name "Discourse CI"
|
|
|
|
|
|
|
|
|
|
- name: Get yarn cache directory
|
|
|
|
|
id: yarn-cache-dir
|
2022-11-11 21:12:08 +08:00
|
|
|
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
|
2022-01-19 18:41:52 +08:00
|
|
|
|
|
|
|
|
- name: Yarn cache
|
2024-01-22 18:50:56 +08:00
|
|
|
uses: actions/cache@v4
|
2022-01-19 18:41:52 +08:00
|
|
|
id: yarn-cache
|
|
|
|
|
with:
|
|
|
|
|
path: ${{ steps.yarn-cache-dir.outputs.dir }}
|
2023-11-10 18:03:15 +08:00
|
|
|
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}-cachev2
|
2022-01-19 18:41:52 +08:00
|
|
|
|
2024-01-10 20:12:36 +08:00
|
|
|
- name: Downgrade Ember
|
|
|
|
|
if: matrix.updated_ember == false
|
|
|
|
|
run: script/switch_ember_version 3
|
DEV: Introduce feature-flag for Ember 5 upgrade
This commit introduces the scaffolding for us to easily switch between Ember 3.28 and Ember 5 on the `main` branch of Discourse. Unfortunately, there is no built-in system to apply this kind of flagging within yarn / ember-cli. There are projects like `ember-try` which are designed for running against multiple version of a dependency, but they do not allow us to 'lock' dependency/sub-dependency versions, and are therefore unsuitable for our use in production.
Instead, we will be maintaining two root `package.json` files, and two `yarn.lock` files. For ember-3, they remain as-is. For ember5, we use a yarn 'resolution' to override the version for ember-source across the entire yarn workspace.
To allow for easy switching with minimal diff against the repository, `package.json` and `yarn.lock` are symlinks which point to `package-ember3.json` and `yarn-ember3.lock` by default. To switch to Ember 5, we can run `script/switch ember version 5` to update the symlinks to point to `package-ember5.json` and `package-ember3.json` respectively. In production, and when using `bin/ember-cli` for development, the ember version can also be upgraded using the `EMBER_VERSION=5` environment variable.
When making changes to dependencies, these should be made against the default `ember3` versions, and then `script/regen_ember_5_lockfile` should be used to regenerate `yarn-ember5.lock` accordingly. A new 'Ember Version Lockfiles' GitHub workflow will automate this process on Dependabot PRs.
When running a local environment against Ember 5, the two symlink changes will show up as git diffs. To avoid us accidentally committing/pushing that change, another GitHub workflow is introduced which checks the default Ember version and raises an error if it is greater than v3.
Supporting two ember versions simultaneously obviously carries significant overhead, so our aim will be to get themes/plugins updated as quickly as possible, and then drop this flag.
2023-11-27 23:52:17 +08:00
|
|
|
|
2022-01-19 18:41:52 +08:00
|
|
|
- name: Yarn install
|
|
|
|
|
working-directory: ./app/assets/javascripts/discourse
|
2023-05-02 23:25:22 +08:00
|
|
|
run: yarn install --frozen-lockfile
|
2022-01-19 18:41:52 +08:00
|
|
|
|
|
|
|
|
- name: Ember Build
|
|
|
|
|
working-directory: ./app/assets/javascripts/discourse
|
|
|
|
|
run: |
|
2022-09-22 01:13:13 +08:00
|
|
|
mkdir /tmp/emberbuild
|
2023-05-12 20:00:04 +08:00
|
|
|
yarn ember build --environment=test -o /tmp/emberbuild
|
2022-01-19 18:41:52 +08:00
|
|
|
|
2022-09-22 01:13:13 +08:00
|
|
|
- name: Core QUnit
|
2022-01-19 18:41:52 +08:00
|
|
|
working-directory: ./app/assets/javascripts/discourse
|
2023-05-12 20:00:04 +08:00
|
|
|
run: yarn ember exam --path /tmp/emberbuild --load-balance --parallel=5 --launch "${{ env.TESTEM_BROWSER }}" --write-execution-file --random
|
2022-08-14 23:30:15 +08:00
|
|
|
timeout-minutes: 15
|
2022-01-19 18:41:52 +08:00
|
|
|
|
2023-12-18 18:47:22 +08:00
|
|
|
- uses: actions/upload-artifact@v4
|
2022-09-22 01:13:13 +08:00
|
|
|
if: ${{ always() }}
|
|
|
|
|
with:
|
2023-12-18 18:47:22 +08:00
|
|
|
name: ember-exam-execution-${{ matrix.browser }}-${{ hashFiles('./app/assets/javascripts/discourse/test-execution-*.json') }}
|
2022-09-22 01:13:13 +08:00
|
|
|
path: ./app/assets/javascripts/discourse/test-execution-*.json
|