From 9caba30d5c9adb81c11185e67b986d13545f5152 Mon Sep 17 00:00:00 2001 From: Alan Guo Xiang Tan Date: Thu, 7 Sep 2023 13:46:23 +0800 Subject: [PATCH] DEV: Add `docker:test:setup` Rake task (#23430) ## What is the context here? The `docker.rake` Rakefile contains Rake tasks that are meant to be run in the `discourse/discourse_test:release` Docker image. For example, we have the `docker:test` Rake task that makes it easier to run the test suite for a particular Discourse commit. Why are we introducing a `docker:test:setup` Rake task? While we have the `docker:test` Rake task, it is very limited in the test commands that can be executed. It is very useful for automated testing but not very useful for running tests in the development environment. Therefore, we are introducing a `docker:test:setup` rake task that can be used to set up the test environment for running tests. The envisioned example usage is something like this: ``` docker run -d --name=discourse_test --entrypoint=/sbin/boot discourse/discourse_test:release docker exec -u discourse:discourse discourse_test ruby script/docker_test.rb --no-tests docker exec -u discourse:discourse discourse_test bundle exec rake docker:test:setup docker exec -u discourse:discourse discourse_test bundle exec rspec ``` --- lib/tasks/docker.rake | 110 +++++++++++++++++++++------------------- script/docker_test.rb | 67 +++++++++++++++++------- script/start_test_db.rb | 4 +- 3 files changed, 111 insertions(+), 70 deletions(-) diff --git a/lib/tasks/docker.rake b/lib/tasks/docker.rake index 04a8018a20d..ce0792e67fc 100644 --- a/lib/tasks/docker.rake +++ b/lib/tasks/docker.rake @@ -1,8 +1,51 @@ # frozen_string_literal: true -# rake docker:test is designed to be used inside the discourse/docker_test image -# running it anywhere else will likely fail -# +# The Rake tasks in this file are designed to be used inside the `discourse/discourse_test:release` image. +# Running it anywhere else is not supported. + +def run_or_fail(command) + log(command) + pid = Process.spawn(command) + Process.wait(pid) + $?.exitstatus == 0 +end + +def log(message) + puts "[#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}] #{message}" +end + +def setup_postgres(skip_init:) + unless skip_init + log "Initializing postgres" + system("script/start_test_db.rb --skip-run", exception: true) + end + + log "Starting postgres" + Process.spawn("script/start_test_db.rb --skip-setup --exec") +end + +def setup_redis + log "Starting background redis" + data_directory = "#{Rails.root}/tmp/test_data/redis" + `rm -rf #{data_directory} && mkdir -p #{data_directory}` + Process.spawn("redis-server --dir #{data_directory} --port 1234") +end + +def migrate_databases(parallel: false, load_plugins: false) + migrate_env = load_plugins ? "LOAD_PLUGINS=1" : "LOAD_PLUGINS=0" + + success = run_or_fail("#{migrate_env} bundle exec rake db:migrate") + success &&= run_or_fail("#{migrate_env} bundle exec rake parallel:migrate") if parallel + success +end + +desc "Setups up the test environment" +task "docker:test:setup" do + setup_redis + setup_postgres(skip_init: false) + migrate_databases(parallel: true, load_plugins: true) +end + # Environment Variables (specific to this rake task) # => SKIP_LINT set to 1 to skip linting (eslint and rubocop) # => SKIP_TESTS set to 1 to skip all tests @@ -33,30 +76,18 @@ # docker run -e SKIP_CORE=1 -v $(pwd)/my-awesome-plugin:/var/www/discourse/plugins/my-awesome-plugin discourse/discourse_test:release # Run tests for a specific plugin (with a plugin mounted from host filesystem): # docker run -e SKIP_CORE=1 SINGLE_PLUGIN='my-awesome-plugin' -v $(pwd)/my-awesome-plugin:/var/www/discourse/plugins/my-awesome-plugin discourse/discourse_test:release - -def run_or_fail(command) - log(command) - pid = Process.spawn(command) - Process.wait(pid) - $?.exitstatus == 0 -end - -def run_or_fail_prettier(*patterns) - if patterns.any? { |p| Dir[p].any? } - patterns = patterns.map { |p| "'#{p}'" }.join(" ") - run_or_fail("yarn pprettier --list-different #{patterns}") - else - puts "Skipping prettier. Pattern not found." - true - end -end - -def log(message) - puts "[#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}] #{message}" -end - desc "Run all tests (JS and code in a standalone environment)" task "docker:test" do + def run_or_fail_prettier(*patterns) + if patterns.any? { |p| Dir[p].any? } + patterns = patterns.map { |p| "'#{p}'" }.join(" ") + run_or_fail("yarn pprettier --list-different #{patterns}") + else + puts "Skipping prettier. Pattern not found." + true + end + end + begin @good = true @good &&= run_or_fail("yarn install") @@ -120,19 +151,8 @@ task "docker:test" do end unless ENV["SKIP_TESTS"] - puts "Cleaning up old test tmp data in tmp/test_data" - `rm -fr tmp/test_data && mkdir -p tmp/test_data/redis && mkdir tmp/test_data/pg` - - puts "Starting background redis" - @redis_pid = Process.spawn("redis-server --dir tmp/test_data/redis") - - unless ENV["SKIP_DB_CREATE"] - puts "Initializing postgres" - system("script/start_test_db.rb --skip-run", exception: true) - end - - puts "Starting postgres" - @pg_pid = Process.spawn("script/start_test_db.rb --skip-setup --exec") + @redis_pid = setup_redis + @pg_pid = setup_postgres(skip_init: ENV["SKIP_DB_CREATE"].present?) ENV["RAILS_ENV"] = "test" # this shaves all the creation of the multisite db off @@ -159,19 +179,7 @@ task "docker:test" do end end - command_prefix = - if ENV["SKIP_PLUGINS"] - # Make sure not to load plugins. bin/rake will add LOAD_PLUGINS=1 automatically unless we set it to 0 explicitly - "LOAD_PLUGINS=0 " - else - "LOAD_PLUGINS=1 " - end - - @good &&= run_or_fail("#{command_prefix}bundle exec rake db:migrate") - - if ENV["USE_TURBO"] - @good &&= run_or_fail("#{command_prefix}bundle exec rake parallel:migrate") - end + @good &&= migrate_databases(parallel: ENV["USE_TURBO"], load_plugins: !ENV["SKIP_PLUGINS"]) unless ENV["JS_ONLY"] if ENV["WARMUP_TMP_FOLDER"] diff --git a/script/docker_test.rb b/script/docker_test.rb index 1c50dd7cf55..5b3057a56d4 100644 --- a/script/docker_test.rb +++ b/script/docker_test.rb @@ -1,12 +1,44 @@ # frozen_string_literal: true -# This script is run in the discourse_test docker image -# Available environment variables: -# => NO_UPDATE disables updating the source code within the discourse_test docker image -# => COMMIT_HASH used by the discourse_test docker image to load a specific commit of discourse -# this can also be set to a branch, e.g. "origin/tests-passed" -# => RUN_SMOKE_TESTS executes the smoke tests instead of the regular tests from docker.rake -# See lib/tasks/docker.rake and lib/tasks/smoke_test.rake for more information +# This script is to be run in the `discourse/discourse_test:release` docker image. + +require "optparse" + +options = {} + +OptionParser + .new do |opts| + opts.banner = "Usage: ruby script/docker_test.rb [options]" + + opts.on( + "--checkout-ref CHECKOUT_REF", + "Checks out the working tree to a specified commit hash or branch. If not specified, defaults to 'origin/tests-passed'.", + ) { |v| options[:checkout_ref] = v } + + opts.on( + "--run-smoke-tests", + "Executes the smoke tests instead of the regular tests from docker.rake. See lib/tasks/smoke_test.rake for more information.", + ) { options[:run_smoke_tests] = true } + + opts.on( + "--no-checkout", + "Does not check out the working tree when this option is passed. By default, the working tree is checked out to the latest commit on the 'origin/tests-passed' branch.", + ) { options[:no_checkout] = true } + + opts.on("--no-tests", "Does not execute any tests") { options[:no_tests] = true } + + opts.on_tail("-h", "--help", "Displays usage information") do + puts opts + exit + end + end + .parse! + +no_checkout = options.has_key?(:no_checkout) ? options[:no_checkout] : ENV["NO_UPDATE"] +checkout_ref = options.has_key?(:checkout_ref) ? options[:checkout_ref] : ENV["COMMIT_HASH"] +run_smoke_test = + options.has_key?(:run_smoke_test) ? options[:run_smoke_test] : ENV["RUN_SMOKE_TESTS"] +no_tests = options.has_key?(:no_tests) ? options[:no_tests] : false def log(message) puts "[#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}] #{message}" @@ -19,20 +51,19 @@ def run_or_fail(command) exit 1 unless $?.exitstatus == 0 end -unless ENV["NO_UPDATE"] +unless no_checkout run_or_fail("git reset --hard") run_or_fail("git fetch") - - checkout = ENV["COMMIT_HASH"] || "origin/tests-passed" - run_or_fail("LEFTHOOK=0 git checkout #{checkout}") - + run_or_fail("LEFTHOOK=0 git checkout #{checkout_ref || "origin/tests-passed"}") run_or_fail("bundle") end -log("Running tests") - -if ENV["RUN_SMOKE_TESTS"] - run_or_fail("bundle exec rake smoke:test") -else - run_or_fail("bundle exec rake docker:test") +unless no_tests + if run_smoke_tests + log("Running smoke tests") + run_or_fail("bundle exec rake smoke:test") + else + log("Running tests") + run_or_fail("bundle exec rake docker:test") + end end diff --git a/script/start_test_db.rb b/script/start_test_db.rb index f9a4d871c84..259fbdf9ec2 100755 --- a/script/start_test_db.rb +++ b/script/start_test_db.rb @@ -24,6 +24,8 @@ while a = ARGV.pop end if should_setup + run "rm -rf #{DATA}" + run "mkdir -p #{DATA}" run "#{BIN}/initdb -D #{DATA}" run "echo fsync = off >> #{DATA}/postgresql.conf" @@ -32,7 +34,7 @@ if should_setup end if should_exec - exec "#{BIN}/postmaster -D #{DATA}" + exec "#{BIN}/postgres -D #{DATA}" elsif should_run run "#{BIN}/pg_ctl -D #{DATA} start" end