mirror of
https://github.com/discourse/discourse.git
synced 2024-11-22 14:38:17 +08:00
2a1952d9ba
Why this change? Pull requests can introduce flaky tests into the mix and we do not want to be hiing that during the pull request process. While this does mean builds for PR will be less stable than the `main` branch without retries, we do not foresee this to be a problem long term since the monitoring of flaky tests on the `main` branch will mean that the number of flaky tests will eventually be reduced. What does this change do? 1. Introduce the `DISCOURSE_TURBO_RSPEC_RETRY_AND_LOG_FLAKY_TESTS` env variable which will initialize `TurboTest::Runner` with the `retry_and_log_flaky_tests` kwarg set to true when set. 2. Change the tests workflow run to set `DISCOURSE_TURBO_RSPEC_RETRY_AND_LOG_FLAKY_TESTS` only when the build type is `backend` or `system` and the `github.ref_name` is `main`.
134 lines
3.5 KiB
Ruby
Executable File
134 lines
3.5 KiB
Ruby
Executable File
#!/usr/bin/env ruby
|
|
# frozen_string_literal: true
|
|
|
|
ENV["RAILS_ENV"] ||= "test"
|
|
|
|
require "./lib/turbo_tests"
|
|
require "optparse"
|
|
|
|
requires = []
|
|
formatters = []
|
|
verbose = false
|
|
fail_fast = nil
|
|
seed = rand(2**16)
|
|
profile = false
|
|
profile_print_slowest_examples_count = 10
|
|
use_runtime_info = nil
|
|
enable_system_tests = nil
|
|
disable_system_tests = nil
|
|
retry_and_log_flaky_tests = !!ENV["DISCOURSE_TURBO_RSPEC_RETRY_AND_LOG_FLAKY_TESTS"]
|
|
|
|
OptionParser
|
|
.new do |opts|
|
|
opts.on("-r", "--require PATH", "Require a file.") { |filename| requires << filename }
|
|
|
|
opts.on("-f", "--format FORMATTER", "Choose a formatter.") do |name|
|
|
formatters << { name: name, outputs: [] }
|
|
end
|
|
|
|
opts.on("-o", "--out FILE", "Write output to a file instead of $stdout") do |filename|
|
|
formatters << { name: "progress", outputs: [] } if formatters.empty?
|
|
formatters.last[:outputs] << filename
|
|
end
|
|
|
|
opts.on("-v", "--verbose", "More output") { verbose = true }
|
|
|
|
opts.on(
|
|
"-p",
|
|
"--profile=[COUNT]",
|
|
"Benchmark the runtime of each example and list the slowest examples (default: 10)",
|
|
) do |count|
|
|
profile = true
|
|
profile_print_slowest_examples_count = count.to_i if count
|
|
end
|
|
|
|
opts.on("--fail-fast=[N]") do |n|
|
|
n =
|
|
begin
|
|
Integer(n)
|
|
rescue StandardError
|
|
nil
|
|
end
|
|
fail_fast = (n.nil? || n < 1) ? 1 : n
|
|
end
|
|
|
|
opts.on(
|
|
"--retry-and-log-flaky-tests",
|
|
"Retry failed tests and log if test is deemed to be flaky",
|
|
) { retry_and_log_flaky_tests = true }
|
|
|
|
opts.on("--seed SEED", "The seed for the random order") { |s| seed = s.to_i }
|
|
|
|
opts.on("--use-runtime-info", "Use runtime info for tests group splitting") do
|
|
use_runtime_info = true
|
|
end
|
|
|
|
opts.on("--enable-system-tests", "Run the system tests (defaults false)") do
|
|
enable_system_tests = true
|
|
end
|
|
|
|
opts.on("--disable-system-tests", "Don't run the system tests (defaults true)") do
|
|
disable_system_tests = true
|
|
end
|
|
end
|
|
.parse!(ARGV)
|
|
|
|
if enable_system_tests && disable_system_tests
|
|
STDERR.puts "Error: I'm not sure how to enable and disable system tests simultaneously"
|
|
exit 1
|
|
end
|
|
|
|
# OptionParser modifies ARGV, leaving the leftover arguments in ARGV
|
|
if ARGV.empty?
|
|
if !enable_system_tests && !disable_system_tests
|
|
STDERR.puts "Not running system tests since it wasn't explicitly specified"
|
|
end
|
|
|
|
run_system_tests = !!enable_system_tests
|
|
|
|
files =
|
|
if run_system_tests
|
|
["#{Rails.root}/spec"]
|
|
else
|
|
TurboTests::Runner.default_spec_folders
|
|
end
|
|
|
|
use_runtime_info = true if use_runtime_info.nil?
|
|
else
|
|
if enable_system_tests || disable_system_tests
|
|
STDERR.puts "Ignoring system test options since files were specified"
|
|
end
|
|
|
|
files = ARGV
|
|
use_runtime_info = false if use_runtime_info.nil?
|
|
end
|
|
|
|
requires.each { |f| require(f) }
|
|
|
|
formatters << { name: "progress", outputs: [] } if formatters.empty?
|
|
|
|
formatters.each { |formatter| formatter[:outputs] << "-" if formatter[:outputs].empty? }
|
|
|
|
puts "::group::Run turbo_rspec" if ENV["GITHUB_ACTIONS"]
|
|
puts "Running turbo_rspec (seed: #{seed}) using files in: #{files}"
|
|
puts "::endgroup::" if ENV["GITHUB_ACTIONS"]
|
|
|
|
success =
|
|
TurboTests::Runner.run(
|
|
formatters: formatters,
|
|
files: files,
|
|
verbose: verbose,
|
|
fail_fast: fail_fast,
|
|
use_runtime_info: use_runtime_info,
|
|
seed: seed.to_s,
|
|
profile:,
|
|
profile_print_slowest_examples_count:,
|
|
retry_and_log_flaky_tests:,
|
|
)
|
|
|
|
if success
|
|
exit 0
|
|
else
|
|
exit 1
|
|
end
|