github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2024-11-22 12:42:16 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

DEV: Add converter framework for migrations-tooling (#28540)

Browse Source 
* Updates GitHub Actions
* Switches from `bundler/inline` to an optional group in the `Gemfile` because the previous solution didn't work well with rspec
* Adds the converter framework and tests
* Allows loading private converters (see README)
* Switches from multiple CLI tools to a single CLI
* Makes DB connections reusable and adds a new abstraction for the `IntermediateDB`
* `IntermediateDB` acts as an interface for IPC calls when a converter steps runs in parallel (forks). Only the main process writes to the DB.
* Includes a simple example implementation of a converter for now.
This commit is contained in:
Gerhard Schlager 2024-09-09 17:14:39 +02:00 committed by GitHub
parent d6eb0f4d96
commit 7c3a29c9d6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
82 changed files with 2787 additions and 456 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.github/dependabot.yml vendored
View File

@ -78,11 +78,3 @@ updates:
types:
patterns:
- "@types/*"
# - package-ecosystem: "bundler"
# directory: "migrations/config/gemfiles/convert"
# schedule:
# interval: "weekly"
# day: "wednesday"
# time: "10:00"
# timezone: "Europe/Vienna"
# versioning-strategy: "increase"

40
.github/workflows/migration-tests.yml vendored
View File

@ -37,7 +37,7 @@ jobs:
fail-fast: false
matrix:
ruby: ["3.2"]
ruby: ["3.3"]
steps:
- name: Set working directory owner
@ -78,7 +78,7 @@ jobs:
${{ steps.container-envs.outputs.ruby_version }}-
${{ steps.container-envs.outputs.debian_release }}-
${{ hashFiles('**/Gemfile.lock') }}-
${{ hashFiles('migrations/config/gemfiles/**/Gemfile') }}
migrations-tooling
- name: Setup gems
run: |
@ -86,8 +86,9 @@ jobs:
bundle config --local path vendor/bundle
bundle config --local deployment true
bundle config --local without development
bundle config --local with migrations
bundle install --jobs $(($(nproc) - 1))
# don't call `bundle clean` clean, we need the gems for the migrations
bundle clean
- name: pnpm install
run: pnpm install --frozen-lockfile
@ -133,36 +134,3 @@ jobs:
- name: RSpec
run: bin/rspec --default-path migrations/spec
runtime:
if: github.event_name == 'pull_request' || github.repository != 'discourse/discourse-private-mirror'
name: Runs on ${{ matrix.os }}, Ruby ${{ matrix.ruby }}
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
os: ["ubuntu-latest", "macos-latest"]
ruby: ["3.2", "3.3"]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 1
- name: Modify path for libpq
if: matrix.os == 'macos-latest'
run: echo "/opt/homebrew/opt/libpq/bin" >> $GITHUB_PATH
- name: Setup Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: ${{ matrix.ruby }}
bundler-cache: true
- name: Run converter
working-directory: migrations
run: bin/convert version

16
Gemfile
View File

@ -274,3 +274,19 @@ gem "csv", require: false
# dependencies for the automation plugin
gem "iso8601"
gem "rrule"
group :migrations, optional: true do
gem "extralite-bundle", require: "extralite"
# auto-loading
gem "zeitwerk"
# databases
gem "trilogy"
# CLI
gem "ruby-progressbar"
# additional Gemfiles from converters
Dir[File.expand_path("migrations/**/Gemfile", __dir__)].each { |path| eval_gemfile(path) }
end

8
Gemfile.lock
View File

@ -137,6 +137,7 @@ GEM
excon (0.111.0)
execjs (2.9.1)
exifr (1.4.0)
extralite-bundle (2.8.2)
fabrication (2.31.0)
faker (2.23.0)
i18n (>= 1.8.11, < 2)
@ -547,6 +548,7 @@ GEM
test-prof (1.4.1)
thor (1.3.2)
timeout (0.4.1)
trilogy (2.8.1)
tzinfo (2.0.6)
concurrent-ruby (~> 1.0)
tzinfo-data (1.2024.2)
@ -624,6 +626,7 @@ DEPENDENCIES
email_reply_trimmer
excon
execjs
extralite-bundle
fabrication
faker (~> 2.16)
fakeweb
@ -704,6 +707,7 @@ DEPENDENCIES
rtlcss
rubocop-discourse
ruby-prof
ruby-progressbar
ruby-readability
rubyzip
sanitize
@ -722,6 +726,7 @@ DEPENDENCIES
syntax_tree-disable_ternary
test-prof
thor
trilogy
tzinfo-data
uglifier
unf
@ -730,6 +735,7 @@ DEPENDENCIES
webmock
yaml-lint
yard
zeitwerk
BUNDLED WITH
2.5.9
2.5.18

4
migrations/.gitignore vendored
View File

@ -1,4 +1,6 @@
!/db/schema/*.sql
!/db/**/*.sql
!/spec/support/fixtures/**/*.sql
tmp/*
private/
Gemfile.lock

24
migrations/README.md
View File

@ -1,7 +1,31 @@
# Migrations Tooling
## Command line interface
```bash
./bin/cli help
```
## Converters
Public converters are stored in `lib/converters/`.
If you need to run a private converter, put its code into a subdirectory of `private/converters/`
## Development
### Installing gems
```bash
bundle config set --local with migrations
bundle install
```
### Updating gems
```bash
bundle update --group migrations
```
### Running tests
You need to execute `rspec` in the root of the project.

23
migrations/bin/cli Executable file
View File

@ -0,0 +1,23 @@
#!/usr/bin/env ruby
# frozen_string_literal: true
require "thor"
require_relative "../lib/migrations"
module Migrations
load_rails_environment
configure_zeitwerk
enable_i18n
class CommandLineInterface < Thor
include ::Migrations::CLI::ConvertCommand
include ::Migrations::CLI::ImportCommand
include ::Migrations::CLI::UploadCommand
def self.exit_on_failure?
true
end
end
CommandLineInterface.start
end

32
migrations/bin/convert
View File

@ -1,32 +0,0 @@
#!/usr/bin/env ruby
# frozen_string_literal: true
require_relative "../lib/migrations"
module Migrations
load_gemfiles("common")
configure_zeitwerk("lib/common", "lib/converters")
module Convert
class CLI < Thor
desc "execute", "Run the conversion"
def execute
FileUtils.mkdir_p("/tmp/converter")
::Migrations::IntermediateDatabaseMigrator.reset!("/tmp/converter/intermediate.db")
::Migrations::IntermediateDatabaseMigrator.migrate("/tmp/converter/intermediate.db")
# require_relative "converters/pepper/main"
end
desc "version", "Print the version"
def version
puts "0.0.1"
end
end
end
end
Migrations::Convert::CLI.start(ARGV)

22
migrations/bin/import
View File

@ -1,22 +0,0 @@
#!/usr/bin/env ruby
# frozen_string_literal: true
require_relative "../lib/migrations"
module Migrations
load_rails_environment
load_gemfiles("common")
configure_zeitwerk("lib/common")
module Import
class << self
def run
puts "Importing into Discourse #{Discourse::VERSION::STRING}"
puts "Extralite SQLite version: #{Extralite.sqlite3_version}"
end
end
end
end
Migrations::Import.run

22
migrations/config/gemfiles/README.md
View File

@ -1,22 +0,0 @@
## Gemfiles for migrations-tooling
This directory contains Gemfiles for the migration related tools.
Those tools use `bundler/inline`, so this isn't strictly needed. However, we use GitHub's Dependabot to keep the
dependencies up-to-date, and it requires a Gemfile to work. Also, it's easier to test the tools with a Gemfile.
Please add an entry in the `.github/workflows/dependabot.yml` file when you add a new Gemfile to enable Dependabot for
the Gemfile.
#### Example
```yaml
- package-ecosystem: "bundler"
directory: "migrations/config/gemfiles/convert"
schedule:
interval: "weekly"
day: "wednesday"
time: "10:00"
timezone: "Europe/Vienna"
versioning-strategy: "increase"
```

30
migrations/config/gemfiles/common/Gemfile
View File

@ -1,30 +0,0 @@
# frozen_string_literal: true
source "https://rubygems.org"
# the minimal Ruby version required by migration-tooling
ruby ">= 3.2.2"
# `activesupport` gem needs to be in sync with the Rails version of Discourse, see `/Gemfile`
lock_file = %w[Gemfile.lock ../Gemfile.lock].detect { File.exist?(_1) }
activesupport_version =
Bundler::LockfileParser
.new(Bundler.read_file(lock_file))
.specs
.detect { _1.name == "activesupport" }
.version
gem "activesupport", "= #{activesupport_version}", require: "active_support"
# for SQLite
gem "extralite-bundle", "~> 2.8", require: "extralite", github: "digital-fabric/extralite"
gem "lru_redux", "~> 1.1", require: false
# for communication between process forks
gem "msgpack", "~> 1.7"
# for CLI
gem "colored2", "~> 4.0"
gem "thor", "~> 1.3"
# auto-loading
gem "zeitwerk", "~> 2.6"

19
migrations/config/locales/migrations.en.yml Normal file
View File

@ -0,0 +1,19 @@
en:
progressbar:
warnings:
one: "%{count} warning"
other: "%{count} warnings"
errors:
one: "%{count} error"
other: "%{count} errors"
estimated: "ETA: %{duration}"
elapsed: "Time: %{duration}"
processed:
percentage: "Processed: %{percentage}"
progress: "Processed: %{current}"
progress_with_max: "Processed: %{current} / %{max}"
converter:
default_step_title: "Converting %{type}"
max_progress_calculation: "Calculating items took %{duration}"

0
migrations/db/schema/002-config.sql → migrations/db/intermediate_db_schema/001-config.sql
View File

0
migrations/db/schema/003-log_entries.sql → migrations/db/intermediate_db_schema/002-log_entries.sql
View File

0
migrations/db/schema/100-base-schema.sql → migrations/db/intermediate_db_schema/100-base-schema.sql
View File

6
migrations/db/schema/001-schema_migrations.sql
View File

@ -1,6 +0,0 @@
CREATE TABLE schema_migrations
(
path TEXT NOT NULL PRIMARY KEY,
created_at DATETIME NOT NULL,
sql_hash TEXT NOT NULL
);

0
migrations/lib/converters/.gitkeep → migrations/db/uploads_db_schema/100-base-schema.sql
View File

52
migrations/lib/cli/convert_command.rb Normal file
View File

@ -0,0 +1,52 @@
# frozen_string_literal: true
module Migrations::CLI::ConvertCommand
def self.included(thor)
thor.class_eval do
desc "convert [FROM]", "Convert a file"
option :settings, type: :string, desc: "Path of settings file", banner: "path"
option :reset, type: :boolean, desc: "Reset database before converting data"
def convert(converter_type)
converter_type = converter_type.downcase
validate_converter_type!(converter_type)
settings = load_settings(converter_type)
::Migrations::Database.reset!(settings[:intermediate_db][:path]) if options[:reset]
converter = "migrations/converters/#{converter_type}/converter".camelize.constantize
converter.new(settings).run
end
private
def validate_converter_type!(type)
converter_names = ::Migrations::Converters.names
raise Thor::Error, <<~MSG if !converter_names.include?(type)
Unknown converter name: #{type}
Valid names are: #{converter_names.join(", ")}
MSG
end
def validate_settings_path!(settings_path)
if !File.exist?(settings_path)
raise Thor::Error, "Settings file not found: #{settings_path}"
end
end
def load_settings(converter_type)
settings_path = calculate_settings_path(converter_type)
validate_settings_path!(settings_path)
YAML.safe_load(File.read(settings_path), symbolize_names: true)
end
def calculate_settings_path(converter_type)
settings_path =
options[:settings] || ::Migrations::Converters.default_settings_path(converter_type)
File.expand_path(settings_path, Dir.pwd)
end
end
end
end

15
migrations/lib/cli/import_command.rb Normal file
View File

@ -0,0 +1,15 @@
# frozen_string_literal: true
module Migrations::CLI::ImportCommand
def self.included(thor)
thor.class_eval do
desc "import", "Import a file"
def import
require "extralite"
puts "Importing into Discourse #{Discourse::VERSION::STRING}"
puts "Extralite SQLite version: #{Extralite.sqlite3_version}"
end
end
end
end

12
migrations/lib/cli/upload_command.rb Normal file
View File

@ -0,0 +1,12 @@
# frozen_string_literal: true
module Migrations::CLI::UploadCommand
def self.included(thor)
thor.class_eval do
desc "upload", "Upload a file"
def upload
puts "Uploading..."
end
end
end
end

22
migrations/lib/common/date_helper.rb Normal file
View File

@ -0,0 +1,22 @@
# frozen_string_literal: true
module Migrations
module DateHelper
# based on code from https://gist.github.com/emmahsax/af285a4b71d8506a1625a3e591dc993b
def self.human_readable_time(secs)
return "< 1 second" if secs < 1
[[60, :seconds], [60, :minutes], [24, :hours], [Float::INFINITY, :days]].map do |count, name|
next if secs <= 0
secs, number = secs.divmod(count)
unless number.to_i == 0
"#{number.to_i} #{number == 1 ? name.to_s.delete_suffix("s") : name}"
end
end
.compact
.reverse
.join(", ")
end
end
end

119
migrations/lib/common/extended_progress_bar.rb Normal file
View File

@ -0,0 +1,119 @@
# frozen_string_literal: true
require "ruby-progressbar"
module Migrations
class ExtendedProgressBar
def initialize(
max_progress: nil,
report_progress_in_percent: false,
use_custom_progress_increment: false
)
@max_progress = max_progress
@report_progress_in_percent = report_progress_in_percent
@use_custom_progress_increment = use_custom_progress_increment
@warning_count = 0
@error_count = 0
@extra_information = ""
@base_format = nil
@progressbar = nil
end
def run
raise "ProgressBar already started" if @progressbar
format = setup_progressbar
yield self
finalize_progressbar(format)
nil
end
def update(stats)
extra_information_changed = false
if stats.warning_count > 0
@warning_count += stats.warning_count
extra_information_changed = true
end
if stats.error_count > 0
@error_count += stats.error_count
extra_information_changed = true
end
if extra_information_changed
@extra_information = +""
if @warning_count > 0
@extra_information << " | " <<
I18n.t("progressbar.warnings", count: @warning_count).yellow
end
if @error_count > 0
@extra_information << " | " << I18n.t("progressbar.errors", count: @error_count).red
end
@progressbar.format = "#{@base_format}#{@extra_information}"
end
if @use_custom_progress_increment
@progressbar.progress += stats.progress
else
@progressbar.increment
end
end
private
def setup_progressbar
format =
if @report_progress_in_percent
I18n.t("progressbar.processed.percentage", percentage: "%J%")
elsif @max_progress
I18n.t("progressbar.processed.progress_with_max", current: "%c", max: "%C")
else
I18n.t("progressbar.processed.progress", current: "%c")
end
@base_format = @max_progress ? " %a |%E | #{format}" : " %a | #{format}"
@progressbar =
::ProgressBar.create(
total: @max_progress,
autofinish: false,
projector: {
type: "smoothing",
strength: 0.5,
},
format: @base_format,
throttle_rate: 0.5,
)
format
end
def finalize_progressbar(format)
print "\033[K" # delete the output of progressbar, because it doesn't overwrite longer lines
final_format = @max_progress ? " %a | #{format}" : " %a | #{format}"
@progressbar.format = "#{final_format}#{@extra_information}"
@progressbar.finish
end
end
end
class ProgressBar
module Components
class Time
def estimated_with_label(out_of_bounds_time_format = nil)
I18n.t("progressbar.estimated", duration: estimated(out_of_bounds_time_format))
end
def elapsed_with_label
I18n.t("progressbar.elapsed", duration: elapsed)
end
end
end
end

105
migrations/lib/common/fork_manager.rb Normal file
View File

@ -0,0 +1,105 @@
# frozen_string_literal: true
module Migrations
module ForkManager
@before_fork_hooks = []
@after_fork_parent_hooks = []
@after_fork_child_hooks = []
@execute_parent_forks = true
class << self
def batch_forks
@execute_parent_forks = false
run_before_fork_hooks
yield
run_after_fork_parent_hooks
@execute_parent_forks = true
end
def before_fork(run_once: false, &block)
if block
@before_fork_hooks << { run_once:, block: }
block
end
end
def remove_before_fork_hook(block)
@before_fork_hooks.delete_if { |hook| hook[:block] == block }
end
def after_fork_parent(run_once: false, &block)
if block
@after_fork_parent_hooks << { run_once:, block: }
block
end
end
def remove_after_fork_parent_hook(block)
@after_fork_parent_hooks.delete_if { |hook| hook[:block] == block }
end
def after_fork_child(&block)
if block
@after_fork_child_hooks << { run_once: true, block: }
block
end
end
def remove_after_fork_child_hook(block)
@after_fork_child_hooks.delete_if { |hook| hook[:block] == block }
end
def fork
run_before_fork_hooks if @execute_parent_forks
pid =
Process.fork do
run_after_fork_child_hooks
yield
end
@after_fork_child_hooks.clear
run_after_fork_parent_hooks if @execute_parent_forks
pid
end
def size
@before_fork_hooks.size + @after_fork_parent_hooks.size + @after_fork_child_hooks.size
end
def clear!
@before_fork_hooks.clear
@after_fork_parent_hooks.clear
@after_fork_child_hooks.clear
end
private
def run_before_fork_hooks
run_hooks(@before_fork_hooks)
end
def run_after_fork_parent_hooks
run_hooks(@after_fork_parent_hooks)
cleanup_run_once_hooks(@after_fork_child_hooks)
end
def run_after_fork_child_hooks
run_hooks(@after_fork_child_hooks)
end
def run_hooks(hooks)
hooks.each { |hook| hook[:block].call }
cleanup_run_once_hooks(hooks)
end
def cleanup_run_once_hooks(hooks)
hooks.delete_if { |hook| hook[:run_once] }
end
end
end
end

121
migrations/lib/common/intermediate_database.rb
View File

@ -1,121 +0,0 @@
# frozen_string_literal: true
require "extralite"
require "lru_redux"
module Migrations
class IntermediateDatabase
DEFAULT_JOURNAL_MODE = "wal"
TRANSACTION_BATCH_SIZE = 1000
PREPARED_STATEMENT_CACHE_SIZE = 5
def self.create_connection(path:, journal_mode: DEFAULT_JOURNAL_MODE)
db = ::Extralite::Database.new(path)
db.pragma(
busy_timeout: 60_000, # 60 seconds
journal_mode: journal_mode,
synchronous: "off",
temp_store: "memory",
locking_mode: journal_mode == "wal" ? "normal" : "exclusive",
cache_size: -10_000, # 10_000 pages
)
db
end
def self.connect
db = self.class.new
yield(db)
ensure
db.close if db
end
attr_reader :connection
attr_reader :path
def initialize(path:, journal_mode: DEFAULT_JOURNAL_MODE)
@path = path
@journal_mode = journal_mode
@connection = self.class.create_connection(path: path, journal_mode: journal_mode)
@statement_counter = 0
# don't cache too many prepared statements
@statement_cache = PreparedStatementCache.new(PREPARED_STATEMENT_CACHE_SIZE)
end
def close
if @connection
commit_transaction
@statement_cache.clear
@connection.close
end
@connection = nil
@statement_counter = 0
end
def reconnect
close
@connection = self.class.create_connection(path: @path, journal_mode: @journal_mode)
end
def copy_from(source_db_paths)
commit_transaction
@statement_counter = 0
table_names = get_table_names
insert_actions = { "config" => "OR REPLACE", "uploads" => "OR IGNORE" }
source_db_paths.each do |source_db_path|
@connection.execute("ATTACH DATABASE ? AS source", source_db_path)
table_names.each do |table_name|
or_action = insert_actions[table_name] || ""
@connection.execute(
"INSERT #{or_action} INTO #{table_name} SELECT * FROM source.#{table_name}",
)
end
@connection.execute("DETACH DATABASE source")
end
end
def begin_transaction
return if @connection.transaction_active?
@connection.execute("BEGIN DEFERRED TRANSACTION")
end
def commit_transaction
return unless @connection.transaction_active?
@connection.execute("COMMIT")
end
private
def insert(sql, *parameters)
begin_transaction if @statement_counter == 0
stmt = @statement_cache.getset(sql) { @connection.prepare(sql) }
stmt.execute(*parameters)
if (@statement_counter += 1) > TRANSACTION_BATCH_SIZE
commit_transaction
@statement_counter = 0
end
end
def iso8601(column_name, alias_name = nil)
alias_name ||= column_name.split(".").last
"strftime('%Y-%m-%dT%H:%M:%SZ', #{column_name}) AS #{alias_name}"
end
def get_table_names
@connection.query_splat(<<~SQL)
SELECT name
FROM sqlite_schema
WHERE type = 'table'
AND name NOT LIKE 'sqlite_%'
AND name NOT IN ('schema_migrations', 'config')
SQL
end
end
end

58
migrations/lib/common/intermediate_database_migrator.rb
View File

@ -1,58 +0,0 @@
# frozen_string_literal: true
module Migrations
class IntermediateDatabaseMigrator
class << self
def reset!(path)
[path, "#{path}-wal", "#{path}-shm"].each { |p| FileUtils.rm_f(p) if File.exist?(p) }
end
def migrate(path)
connection = IntermediateDatabase.create_connection(path: path)
performed_migrations = find_performed_migrations(connection)
path = File.join(::Migrations.root_path, "db", "schema")
migrate_from_path(connection, path, performed_migrations)
connection.close
end
private
def new_database?(connection)
connection.query_single_splat(<<~SQL) == 0
SELECT COUNT(*)
FROM sqlite_schema
WHERE type = 'table' AND name = 'schema_migrations'
SQL
end
def find_performed_migrations(connection)
return Set.new if new_database?(connection)
connection.query_splat(<<~SQL).to_set
SELECT path
FROM schema_migrations
SQL
end
def migrate_from_path(connection, migration_path, performed_migrations)
file_pattern = File.join(migration_path, "*.sql")
Dir[file_pattern].sort.each do |path|
relative_path = Pathname(path).relative_path_from(Migrations.root_path).to_s
if performed_migrations.exclude?(relative_path)
sql = File.read(path)
sql_hash = Digest::SHA1.hexdigest(sql)
connection.execute(sql)
connection.execute(<<~SQL, path: relative_path, sql_hash: sql_hash)
INSERT INTO schema_migrations (path, created_at, sql_hash)
VALUES (:path, datetime('now'), :sql_hash)
SQL
end
end
end
end
end
end

45
migrations/lib/converters.rb Normal file
View File

@ -0,0 +1,45 @@
# frozen_string_literal: true
module Migrations
module Converters
def self.all
@all_converters ||=
begin
base_path = File.join(::Migrations.root_path, "lib", "converters", "base")
core_paths = Dir[File.join(::Migrations.root_path, "lib", "converters", "*")]
private_paths = Dir[File.join(::Migrations.root_path, "private", "converters", "*")]
all_paths = core_paths - [base_path] + private_paths
all_paths.each_with_object({}) do |path, hash|
next unless File.directory?(path)
name = File.basename(path).downcase
existing_path = hash[name]
raise <<~MSG if existing_path
Duplicate converter name found: #{name}
* #{existing_path}
* #{path}
MSG
hash[name] = path
end
end
end
def self.names
self.all.keys.sort
end
def self.path_of(converter_name)
converter_name = converter_name.downcase
path = self.all[converter_name]
raise "Could not find a converter named '#{converter_name}'" unless path
path
end
def self.default_settings_path(converter_name)
File.join(path_of(converter_name), "settings.yml")
end
end
end

79
migrations/lib/converters/base/converter.rb Normal file
View File

@ -0,0 +1,79 @@
# frozen_string_literal: true
module Migrations::Converters::Base
class Converter
attr_accessor :settings
def initialize(settings)
@settings = settings
end
def run
if respond_to?(:setup)
puts "Initializing..."
setup
end
create_database
steps.each do |step_class|
step = create_step(step_class)
before_step_execution(step)
execute_step(step)
after_step_execution(step)
end
rescue SignalException
STDERR.puts "\nAborted"
exit(1)
ensure
::Migrations::Database::IntermediateDB.close
end
def steps
raise NotImplementedError
end
def before_step_execution(step)
# do nothing
end
def execute_step(step)
executor =
if step.is_a?(ProgressStep)
ProgressStepExecutor
else
StepExecutor
end
executor.new(step).execute
end
def after_step_execution(step)
# do nothing
end
def step_args(step_class)
{}
end
private
def create_database
db_path = File.expand_path(settings[:intermediate_db][:path], ::Migrations.root_path)
::Migrations::Database.migrate(
db_path,
migrations_path: ::Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH,
)
db = ::Migrations::Database.connect(db_path)
::Migrations::Database::IntermediateDB.setup(db)
end
def create_step(step_class)
default_args = { settings: settings }
args = default_args.merge(step_args(step_class))
step_class.new(args)
end
end
end

32
migrations/lib/converters/base/parallel_job.rb Normal file
View File

@ -0,0 +1,32 @@
# frozen_string_literal: true
module Migrations::Converters::Base
class ParallelJob
def initialize(step)
@step = step
@stats = ProgressStats.new
@offline_connection = ::Migrations::Database::OfflineConnection.new
::Migrations::ForkManager.after_fork_child do
::Migrations::Database::IntermediateDB.setup(@offline_connection)
end
end
def run(item)
@stats.reset!
@offline_connection.clear!
begin
@step.process_item(item, @stats)
rescue StandardError => e
@stats.log_error("Failed to process item", exception: e, details: item)
end
[@offline_connection.parametrized_insert_statements, @stats]
end
def cleanup
end
end
end

47
migrations/lib/converters/base/progress_stats.rb Normal file
View File

@ -0,0 +1,47 @@
# frozen_string_literal: true
module Migrations::Converters::Base
class ProgressStats
attr_accessor :progress, :warning_count, :error_count
def initialize
reset!
end
def reset!
@progress = 1
@warning_count = 0
@error_count = 0
end
def log_info(message, details: nil)
log(::Migrations::Database::IntermediateDB::LogEntry::INFO, message, details:)
end
def log_warning(message, exception: nil, details: nil)
@warning_count += 1
log(::Migrations::Database::IntermediateDB::LogEntry::WARNING, message, exception:, details:)
end
def log_error(message, exception: nil, details: nil)
@error_count += 1
log(::Migrations::Database::IntermediateDB::LogEntry::ERROR, message, exception:, details:)
end
def ==(other)
other.is_a?(ProgressStats) && progress == other.progress &&
warning_count == other.warning_count && error_count == other.error_count
end
private
def log(type, message, exception: nil, details: nil)
::Migrations::Database::IntermediateDB::LogEntry.create!(
type:,
message:,
exception:,
details:,
)
end
end
end

43
migrations/lib/converters/base/progress_step.rb Normal file
View File

@ -0,0 +1,43 @@
# frozen_string_literal: true
module Migrations::Converters::Base
class ProgressStep < Step
def max_progress
nil
end
def items
raise NotImplementedError
end
def process_item(item, stats)
raise NotImplementedError
end
class << self
def run_in_parallel(value)
@run_in_parallel = !!value
end
def run_in_parallel?
@run_in_parallel == true
end
def report_progress_in_percent(value)
@report_progress_in_percent = !!value
end
def report_progress_in_percent?
@report_progress_in_percent == true
end
def use_custom_progress_increment(value)
@use_custom_progress_increment = !!value
end
def use_custom_progress_increment?
@use_custom_progress_increment == true
end
end
end
end

123
migrations/lib/converters/base/progress_step_executor.rb Normal file
View File

@ -0,0 +1,123 @@
# frozen_string_literal: true
require "etc"
require "colored2"
module Migrations::Converters::Base
class ProgressStepExecutor
WORKER_COUNT = Etc.nprocessors - 1 # leave 1 CPU free to do other work
MIN_PARALLEL_ITEMS = WORKER_COUNT * 10
MAX_QUEUE_SIZE = WORKER_COUNT * 100
PRINT_RUNTIME_AFTER_SECONDS = 5
def initialize(step)
@step = step
end
def execute
@max_progress = calculate_max_progress
puts @step.class.title
@step.execute
if execute_in_parallel?
execute_parallel
else
execute_serially
end
end
private
def execute_in_parallel?
@step.class.run_in_parallel? && (@max_progress.nil? || @max_progress > MIN_PARALLEL_ITEMS)
end
def execute_serially
job = SerialJob.new(@step)
with_progressbar do |progressbar|
@step.items.each do |item|
stats = job.run(item)
progressbar.update(stats)
end
end
end
def execute_parallel
worker_output_queue = SizedQueue.new(MAX_QUEUE_SIZE)
work_queue = SizedQueue.new(MAX_QUEUE_SIZE)
workers = start_workers(work_queue, worker_output_queue)
writer_thread = start_db_writer(worker_output_queue)
push_work(work_queue)
workers.each(&:wait)
worker_output_queue.close
writer_thread.join
end
def calculate_max_progress
start_time = Time.now
max_progress = @step.max_progress
duration = Time.now - start_time
if duration > PRINT_RUNTIME_AFTER_SECONDS
message =
I18n.t(
"converter.max_progress_calculation",
duration: ::Migrations::DateHelper.human_readable_time(duration),
)
puts " #{message}"
end
max_progress
end
def with_progressbar
::Migrations::ExtendedProgressBar
.new(
max_progress: @max_progress,
report_progress_in_percent: @step.class.report_progress_in_percent?,
use_custom_progress_increment: @step.class.use_custom_progress_increment?,
)
.run { |progressbar| yield progressbar }
end
def start_db_writer(worker_output_queue)
Thread.new do
Thread.current.name = "writer_thread"
with_progressbar do |progressbar|
while (parametrized_insert_statements, stats = worker_output_queue.pop)
parametrized_insert_statements.each do |sql, parameters|
::Migrations::Database::IntermediateDB.insert(sql, *parameters)
end
progressbar.update(stats)
end
end
end
end
def start_workers(work_queue, worker_output_queue)
workers = []
Process.warmup
::Migrations::ForkManager.batch_forks do
WORKER_COUNT.times do |index|
job = ParallelJob.new(@step)
workers << Worker.new(index, work_queue, worker_output_queue, job).start
end
end
workers
end
def push_work(work_queue)
@step.items.each { |item| work_queue.push(item) }
work_queue.close
end
end
end

25
migrations/lib/converters/base/serial_job.rb Normal file
View File

@ -0,0 +1,25 @@
# frozen_string_literal: true
module Migrations::Converters::Base
class SerialJob
def initialize(step)
@step = step
@stats = ProgressStats.new
end
def run(item)
@stats.reset!
begin
@step.process_item(item, @stats)
rescue StandardError => e
@stats.log_error("Failed to process item", exception: e, details: item)
end
@stats
end
def cleanup
end
end
end

33
migrations/lib/converters/base/step.rb Normal file
View File

@ -0,0 +1,33 @@
# frozen_string_literal: true
module Migrations::Converters::Base
class Step
IntermediateDB = ::Migrations::Database::IntermediateDB
attr_accessor :settings
def initialize(args = {})
args.each { |arg, value| instance_variable_set("@#{arg}", value) if respond_to?(arg, true) }
end
def execute
# do nothing
end
class << self
def title(
value = (
getter = true
nil
)
)
@title = value unless getter
@title.presence ||
I18n.t(
"converter.default_step_title",
type: name&.demodulize&.underscore&.humanize(capitalize: false),
)
end
end
end
end

14
migrations/lib/converters/base/step_executor.rb Normal file
View File

@ -0,0 +1,14 @@
# frozen_string_literal: true
module Migrations::Converters::Base
class StepExecutor
def initialize(step)
@step = step
end
def execute
puts @step.class.title
@step.execute
end
end
end

101
migrations/lib/converters/base/worker.rb Normal file
View File

@ -0,0 +1,101 @@
# frozen_string_literal: true
require "oj"
module Migrations::Converters::Base
class Worker
OJ_SETTINGS = {
mode: :custom,
create_id: "^o",
create_additions: true,
cache_keys: true,
class_cache: true,
symbol_keys: true,
}
def initialize(index, input_queue, output_queue, job)
@index = index
@input_queue = input_queue
@output_queue = output_queue
@job = job
@threads = []
@mutex = Mutex.new
@data_processed = ConditionVariable.new
end
def start
parent_input_stream, parent_output_stream = IO.pipe
fork_input_stream, fork_output_stream = IO.pipe
worker_pid =
start_fork(parent_input_stream, parent_output_stream, fork_input_stream, fork_output_stream)
fork_output_stream.close
parent_input_stream.close
start_input_thread(parent_output_stream, worker_pid)
start_output_thread(fork_input_stream)
self
end
def wait
@threads.each(&:join)
end
private
def start_fork(parent_input_stream, parent_output_stream, fork_input_stream, fork_output_stream)
::Migrations::ForkManager.fork do
begin
Process.setproctitle("worker_process#{@index}")
parent_output_stream.close
fork_input_stream.close
Oj.load(parent_input_stream, OJ_SETTINGS) do |data|
result = @job.run(data)
Oj.to_stream(fork_output_stream, result, OJ_SETTINGS)
end
rescue SignalException
exit(1)
ensure
@job.cleanup
end
end
end
def start_input_thread(output_stream, worker_pid)
@threads << Thread.new do
Thread.current.name = "worker_#{@index}_input"
begin
while (data = @input_queue.pop)
Oj.to_stream(output_stream, data, OJ_SETTINGS)
@mutex.synchronize { @data_processed.wait(@mutex) }
end
ensure
output_stream.close
Process.waitpid(worker_pid)
end
end
end
def start_output_thread(input_stream)
@threads << Thread.new do
Thread.current.name = "worker_#{@index}_output"
begin
Oj.load(input_stream, OJ_SETTINGS) do |data|
@output_queue.push(data)
@mutex.synchronize { @data_processed.signal }
end
ensure
input_stream.close
@mutex.synchronize { @data_processed.signal }
end
end
end
end
end

9
migrations/lib/converters/example/converter.rb Normal file
View File

@ -0,0 +1,9 @@
# frozen_string_literal: true
module Migrations::Converters::Example
class Converter < ::Migrations::Converters::Base::Converter
def steps
[Step1, Step2, Step3, Step4]
end
end
end

2
migrations/lib/converters/example/settings.yml Normal file
View File

@ -0,0 +1,2 @@
intermediate_db:
path: "~/intermediate.db"

12
migrations/lib/converters/example/steps/step1.rb Normal file
View File

@ -0,0 +1,12 @@
# frozen_string_literal: true
module Migrations::Converters::Example
class Step1 < ::Migrations::Converters::Base::Step
title "Hello world"
def execute
super
IntermediateDB::LogEntry.create!(type: "info", message: "This is a test")
end
end
end

20
migrations/lib/converters/example/steps/step2.rb Normal file
View File

@ -0,0 +1,20 @@
# frozen_string_literal: true
module Migrations::Converters::Example
class Step2 < ::Migrations::Converters::Base::ProgressStep
run_in_parallel false
def items
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
end
def process_item(item, stats)
sleep(0.5)
stats.warning_count += 1 if item.in?([3, 7, 9])
stats.error_count += 1 if item.in?([6, 10])
IntermediateDB::LogEntry.create!(type: "info", message: "Step2 - #{item}")
end
end
end

21
migrations/lib/converters/example/steps/step3.rb Normal file
View File

@ -0,0 +1,21 @@
# frozen_string_literal: true
module Migrations::Converters::Example
class Step3 < ::Migrations::Converters::Base::ProgressStep
run_in_parallel true
def max_progress
1000
end
def items
(1..1000).map { |i| { counter: i } }
end
def process_item(item, stats)
sleep(0.5)
IntermediateDB::LogEntry.create!(type: "info", message: "Step3 - #{item[:counter]}")
end
end
end

6
migrations/lib/converters/example/steps/subdirectory/step4.rb Normal file
View File

@ -0,0 +1,6 @@
# frozen_string_literal: true
module Migrations::Converters::Example
class Step4 < ::Migrations::Converters::Base::Step
end
end

7
migrations/lib/converters/pepper/main.rb
View File

@ -1,7 +0,0 @@
# frozen_string_literal: true
gemfile do
source "https://rubygems.org"
gem "hashids"
end

59
migrations/lib/database.rb Normal file
View File

@ -0,0 +1,59 @@
# frozen_string_literal: true
require "date"
require "extralite"
require "ipaddr"
module Migrations
module Database
INTERMEDIATE_DB_SCHEMA_PATH = File.join(::Migrations.root_path, "db", "intermediate_db_schema")
UPLOADS_DB_SCHEMA_PATH = File.join(::Migrations.root_path, "db", "uploads_db_schema")
module_function
def migrate(db_path, migrations_path:)
Migrator.new(db_path).migrate(migrations_path)
end
def reset!(db_path)
Migrator.new(db_path).reset!
end
def connect(path)
connection = Connection.new(path:)
return connection unless block_given?
begin
yield(connection)
ensure
connection.close
end
nil
end
def format_datetime(value)
value&.utc&.iso8601
end
def format_date(value)
value&.to_date&.iso8601
end
def format_boolean(value)
return nil if value.nil?
value ? 1 : 0
end
def format_ip_address(value)
return nil if value.blank?
IPAddr.new(value).to_s
rescue ArgumentError
nil
end
def to_blob(value)
return nil if value.blank?
Extralite::Blob.new(value)
end
end
end

101
migrations/lib/database/connection.rb Normal file
View File

@ -0,0 +1,101 @@
# frozen_string_literal: true
require "extralite"
require "lru_redux"
module Migrations::Database
class Connection
TRANSACTION_BATCH_SIZE = 1000
PREPARED_STATEMENT_CACHE_SIZE = 5
def self.open_database(path:)
FileUtils.mkdir_p(File.dirname(path))
db = Extralite::Database.new(path)
db.pragma(
busy_timeout: 60_000, # 60 seconds
journal_mode: "wal",
synchronous: "off",
temp_store: "memory",
locking_mode: "normal",
cache_size: -10_000, # 10_000 pages
)
db
end
attr_reader :db, :path
def initialize(path:, transaction_batch_size: TRANSACTION_BATCH_SIZE)
@path = path
@transaction_batch_size = transaction_batch_size
@db = self.class.open_database(path:)
@statement_counter = 0
# don't cache too many prepared statements
@statement_cache = PreparedStatementCache.new(PREPARED_STATEMENT_CACHE_SIZE)
@fork_hooks = setup_fork_handling
end
def close
close_connection(keep_path: false)
before_hook, after_hook = @fork_hooks
::Migrations::ForkManager.remove_before_fork_hook(before_hook)
::Migrations::ForkManager.remove_after_fork_parent_hook(after_hook)
end
def closed?
!@db || @db.closed?
end
def insert(sql, parameters = [])
begin_transaction if @statement_counter == 0
stmt = @statement_cache.getset(sql) { @db.prepare(sql) }
stmt.execute(parameters)
if (@statement_counter += 1) >= @transaction_batch_size
commit_transaction
@statement_counter = 0
end
end
private
def begin_transaction
return if @db.transaction_active?
@db.execute("BEGIN DEFERRED TRANSACTION")
end
def commit_transaction
return unless @db.transaction_active?
@db.execute("COMMIT")
end
def close_connection(keep_path:)
return if !@db
commit_transaction
@statement_cache.clear
@db.close
@path = nil unless keep_path
@db = nil
@statement_counter = 0
end
def setup_fork_handling
before_hook = ::Migrations::ForkManager.before_fork { close_connection(keep_path: true) }
after_hook =
::Migrations::ForkManager.after_fork_parent do
@db = self.class.open_database(path: @path) if @path
end
[before_hook, after_hook]
end
end
end

20
migrations/lib/database/intermediate_db.rb Normal file
View File

@ -0,0 +1,20 @@
# frozen_string_literal: true
require "singleton"
module Migrations::Database
module IntermediateDB
def self.setup(db_connection)
close
@db = db_connection
end
def self.insert(sql, *parameters)
@db.insert(sql, parameters)
end
def self.close
@db.close if @db
end
end
end

25
migrations/lib/database/intermediate_db/log_entry.rb Normal file
View File

@ -0,0 +1,25 @@
# frozen_string_literal: true
module Migrations::Database::IntermediateDB
module LogEntry
INFO = "info"
WARNING = "warning"
ERROR = "error"
SQL = <<~SQL
INSERT INTO log_entries (created_at, type, message, exception, details)
VALUES (?, ?, ?, ?, ?)
SQL
def self.create!(created_at: Time.now, type:, message:, exception: nil, details: nil)
::Migrations::Database::IntermediateDB.insert(
SQL,
::Migrations::Database.format_datetime(created_at),
type,
message,
exception&.full_message(highlight: false),
details,
)
end
end
end

82
migrations/lib/database/migrator.rb Normal file
View File

@ -0,0 +1,82 @@
# frozen_string_literal: true
module Migrations::Database
class Migrator
def initialize(db_path)
@db_path = db_path
@db = nil
end
def migrate(migrations_path)
@migrations_path = migrations_path
@db = Connection.open_database(path: @db_path)
if new_database?
create_schema_migrations_table
performed_migrations = Set.new
else
performed_migrations = find_performed_migrations
end
migrate_from_path(@migrations_path, performed_migrations)
@db.close
end
def reset!
[@db_path, "#{@db_path}-wal", "#{@db_path}-shm"].each do |path|
FileUtils.remove_file(path, force: true) if File.exist?(path)
end
end
private
def new_database?
@db.query_single_splat(<<~SQL) == 0
SELECT COUNT(*)
FROM sqlite_schema
WHERE type = 'table' AND name = 'schema_migrations'
SQL
end
def find_performed_migrations
@db.query_splat(<<~SQL).to_set
SELECT path
FROM schema_migrations
SQL
end
def create_schema_migrations_table
@db.execute(<<~SQL)
CREATE TABLE schema_migrations
(
path TEXT NOT NULL PRIMARY KEY,
created_at DATETIME NOT NULL,
sql_hash TEXT NOT NULL
);
SQL
end
def migrate_from_path(migration_path, performed_migrations)
file_pattern = File.join(migration_path, "*.sql")
root_path = @migrations_path || ::Migrations.root_path
Dir[file_pattern].sort.each do |path|
relative_path = Pathname(path).relative_path_from(root_path).to_s
if performed_migrations.exclude?(relative_path)
sql = File.read(path)
sql_hash = Digest::SHA1.hexdigest(sql)
@db.transaction do
@db.execute(sql)
@db.execute(<<~SQL, path: relative_path, sql_hash: sql_hash)
INSERT INTO schema_migrations (path, created_at, sql_hash)
VALUES (:path, datetime('now'), :sql_hash)
SQL
end
end
end
end
end
end

29
migrations/lib/database/offline_connection.rb Normal file
View File

@ -0,0 +1,29 @@
# frozen_string_literal: true
module Migrations::Database
class OfflineConnection
def initialize
@parametrized_insert_statements = []
end
def close
@parametrized_insert_statements = nil
end
def closed?
@parametrized_insert_statements.nil?
end
def insert(sql, parameters = [])
@parametrized_insert_statements << [sql, parameters]
end
def parametrized_insert_statements
@parametrized_insert_statements
end
def clear!
@parametrized_insert_statements.clear if @parametrized_insert_statements
end
end
end

4
migrations/lib/common/prepared_statement_cache.rb → migrations/lib/database/prepared_statement_cache.rb
View File

@ -1,7 +1,7 @@
# frozen_string_literal: true
class Migrations
class PreparedStatementCache < ::LruRedux::Cache
module Migrations::Database
class PreparedStatementCache < LruRedux::Cache
class PreparedStatementHash < Hash
def shift
result = super

96
migrations/lib/migrations.rb
View File

@ -1,61 +1,75 @@
# frozen_string_literal: true
require "bundler/inline"
require "bundler/ui"
require "bundler/setup"
Bundler.setup
require "active_support"
require "active_support/core_ext"
require "zeitwerk"
require_relative "converters"
module Migrations
def self.root_path
@root_path ||= File.expand_path("..", __dir__)
end
def self.load_gemfiles(*relative_paths)
gemfiles_root_path = File.join(Migrations.root_path, "config/gemfiles")
relative_paths.each do |relative_path|
path = File.join(File.expand_path(relative_path, gemfiles_root_path), "Gemfile")
unless File.exist?(path)
warn "Could not find Gemfile at #{path}"
exit 1
end
gemfile_content = File.read(path)
# Create new UI and set level to confirm to avoid printing unnecessary messages
bundler_ui = Bundler::UI::Shell.new
bundler_ui.level = "confirm"
begin
gemfile(true, ui: bundler_ui) do
# rubocop:disable Security/Eval
eval(gemfile_content, nil, path, 1)
# rubocop:enable Security/Eval
end
rescue Bundler::BundlerError => e
warn "\e[31m#{e.message}\e[0m"
exit 1
end
end
end
def self.load_rails_environment(quiet: false)
puts "Loading application..." unless quiet
message = "Loading Rails environment ..."
print message unless quiet
rails_root = File.expand_path("../..", __dir__)
# rubocop:disable Discourse/NoChdir
Dir.chdir(rails_root) { require File.join(rails_root, "config/environment") }
Dir.chdir(rails_root) do
begin
require File.join(rails_root, "config/environment")
rescue LoadError => e
$stderr.puts e.message
raise
end
end
# rubocop:enable Discourse/NoChdir
print "\r"
print " " * message.length
print "\r"
end
def self.configure_zeitwerk(*directories)
require "zeitwerk"
root_path = Migrations.root_path
def self.configure_zeitwerk
loader = Zeitwerk::Loader.new
directories.each do |dir|
loader.push_dir(File.expand_path(dir, root_path), namespace: Migrations)
loader.log! if ENV["DEBUG"]
loader.inflector.inflect(
{ "cli" => "CLI", "intermediate_db" => "IntermediateDB", "uploads_db" => "UploadsDB" },
)
loader.push_dir(File.join(::Migrations.root_path, "lib"), namespace: ::Migrations)
loader.push_dir(File.join(::Migrations.root_path, "lib", "common"), namespace: ::Migrations)
# All sub-directories of a converter should have the same namespace.
# Unfortunately `loader.collapse` doesn't work recursively.
Converters.all.each do |name, converter_path|
module_name = name.camelize.to_sym
namespace = ::Migrations::Converters.const_set(module_name, Module.new)
Dir[File.join(converter_path, "**", "*")].each do |subdirectory|
next unless File.directory?(subdirectory)
loader.push_dir(subdirectory, namespace: namespace)
end
end
loader.setup
end
def self.enable_i18n
require "i18n"
locale_glob = File.join(::Migrations.root_path, "config", "locales", "**", "migrations.*.yml")
I18n.load_path += Dir[locale_glob]
I18n.backend.load_translations
# always use English for now
I18n.default_locale = :en
I18n.locale = :en
end
end

82
migrations/scripts/benchmarks/RESULTS.md Normal file
View File

@ -0,0 +1,82 @@
# Benchmark Results
Here are the latest benchmark results. All benchmarks ran with `ruby 3.2.3 (2024-01-18 revision 52bb2ac0a6) [x86_64-linux]`
## database_write.rb
Compares the INSERT speed of SQLite and DuckDB
| Database | User Time | System Time | Total Time | Real Time |
|-----------|----------:|------------:|-----------:|-----------:|
| SQLite3 | 99.212731 | 4.883932 | 104.096663 | 104.233575 |
| Extralite | 45.396666 | 4.457247 | 49.853913 | 50.065680 |
| DuckDB | 49.012140 | 10.210994 | 59.223134 | 52.095679 |
## hash_vs_data.rb
Compares the INSERT speed when the data is bound as Hash or Data class
```
Extralite regular 868.703k (± 2.2%) i/s - 8.744M in 10.070511s
Extralite hash 579.753k (± 1.2%) i/s - 5.838M in 10.071266s
Extralite data 672.752k (± 0.8%) i/s - 6.790M in 10.093191s
Extralite data/array 826.296k (± 0.9%) i/s - 8.318M in 10.067518s
SQLite3 regular 362.037k (± 0.7%) i/s - 3.628M in 10.021699s
SQLite3 hash 308.647k (± 1.1%) i/s - 3.111M in 10.081159s
SQLite3 data/hash 288.747k (± 2.7%) i/s - 2.890M in 10.018335s
Comparison:
Extralite regular: 868702.8 i/s
Extralite data/array: 826295.7 i/s - 1.05x slower
Extralite data: 672752.0 i/s - 1.29x slower
Extralite hash: 579753.5 i/s - 1.50x slower
SQLite3 regular: 362037.0 i/s - 2.40x slower
SQLite3 hash: 308646.7 i/s - 2.81x slower
SQLite3 data/hash: 288747.1 i/s - 3.01x slower
```
## parameter_binding.rb
A similar benchmark that looks at various parameter binding styles, especially in Extralite
```
Extralite regular 825.159 (± 0.6%) i/s - 8.316k in 10.078450s
Extralite named 571.135 (± 0.4%) i/s - 5.742k in 10.053796s
Extralite index 769.273 (± 1.0%) i/s - 7.742k in 10.065238s
Extralite array 860.549 (± 0.5%) i/s - 8.624k in 10.021749s
SQLite3 regular 361.745 (± 0.6%) i/s - 3.636k in 10.051588s
SQLite3 named 307.875 (± 0.6%) i/s - 3.090k in 10.036954s
Comparison:
Extralite array: 860.5 i/s
Extralite regular: 825.2 i/s - 1.04x slower
Extralite index: 769.3 i/s - 1.12x slower
Extralite named: 571.1 i/s - 1.51x slower
SQLite3 regular: 361.7 i/s - 2.38x slower
SQLite3 named: 307.9 i/s - 2.80x slower
```
## time_formatting.rb
Fastest way of converting `Time` into `String`?
```
Time#iso8601 1.084M (± 0.9%) i/s - 10.875M in 10.033905s
Time#strftime 1.213M (± 1.4%) i/s - 12.200M in 10.056764s
DateTime#iso8601 2.419M (± 1.8%) i/s - 24.296M in 10.046295s
Comparison:
DateTime#iso8601: 2419162.1 i/s
Time#strftime: 1213390.0 i/s - 1.99x slower
Time#iso8601: 1083922.8 i/s - 2.23x slower
```
## write.rb
Compares writing lots of data into a single SQLite database.
```
single writer 43.9766 seconds
forked writer - same DB 53.5112 seconds
forked writer - multi DB 3.0815 seconds
```

2
migrations/scripts/benchmarks/hash_vs_data.rb
View File

@ -6,7 +6,7 @@ require "bundler/inline"
gemfile(true) do
source "https://rubygems.org"
gem "benchmark-ips"
gem "extralite-bundle", github: "digital-fabric/extralite"
gem "extralite-bundle"
gem "sqlite3"
end

2
migrations/scripts/benchmarks/parameter_binding.rb
View File

@ -6,7 +6,7 @@ require "bundler/inline"
gemfile(true) do
source "https://rubygems.org"
gem "benchmark-ips"
gem "extralite-bundle", github: "digital-fabric/extralite"
gem "extralite-bundle"
gem "sqlite3"
end

4
migrations/scripts/benchmarks/write.rb
View File

@ -5,7 +5,7 @@ require "bundler/inline"
gemfile(true) do
source "https://rubygems.org"
gem "extralite-bundle", github: "digital-fabric/extralite"
gem "extralite-bundle"
end
require "etc"
@ -43,7 +43,7 @@ def with_db_path
yield tempfile.path
db = create_extralite_db(tempfile.path)
row_count = db.query_single_value("SELECT COUNT(*) FROM users")
row_count = db.query_single_splat("SELECT COUNT(*) FROM users")
puts "Row count: #{row_count}" if row_count != ROW_COUNT
db.close
ensure

3
migrations/scripts/generate_schema
View File

@ -11,7 +11,6 @@ require_relative "../lib/migrations"
module Migrations
load_rails_environment
load_gemfiles("common")
class SchemaGenerator
def initialize(opts = {})
@ -305,4 +304,4 @@ module Migrations
end
end
Migrations::SchemaGenerator.new(output_file_path: ARGV.first).run
::Migrations::SchemaGenerator.new(output_file_path: ARGV.first).run

0
migrations/spec/.gitkeep
View File

15
migrations/spec/bin/import_spec.rb
View File

@ -1,15 +0,0 @@
# frozen_string_literal: true
RSpec.describe "Migrations::Import" do
subject(:cli) do
# rubocop:disable Discourse/NoChdir
Dir.chdir("migrations") { system("bin/import", exception: true) }
# rubocop:enable Discourse/NoChdir
end
it "works" do
expect { cli }.to output(
include("Importing into Discourse #{Discourse::VERSION::STRING}"),
).to_stdout_from_any_process
end
end

83
migrations/spec/lib/converters/base/parallel_job_spec.rb Normal file
View File

@ -0,0 +1,83 @@
# frozen_string_literal: true
RSpec.describe ::Migrations::Converters::Base::ParallelJob do
subject(:job) { described_class.new(step) }
let(:step) { instance_double(::Migrations::Converters::Base::ProgressStep) }
let(:item) { { key: "value" } }
let(:stats) { instance_double(::Migrations::Converters::Base::ProgressStats) }
let(:intermediate_db) { class_double(::Migrations::Database::IntermediateDB).as_stubbed_const }
before do
allow(::Migrations::Converters::Base::ProgressStats).to receive(:new).and_return(stats)
allow(stats).to receive(:reset!)
allow(stats).to receive(:log_error)
allow(intermediate_db).to receive(:setup)
allow(intermediate_db).to receive(:close)
end
after do
::Migrations::Database::IntermediateDB.setup(nil)
::Migrations::ForkManager.clear!
end
describe "#initialize" do
it "sets up `OfflineConnection` as `IntermediateDB` connection" do
described_class.new(step)
::Migrations::ForkManager.fork do
expect(intermediate_db).to have_received(:setup).with(
an_instance_of(::Migrations::Database::OfflineConnection),
)
end
end
end
describe "#run" do
let(:offline_connection) { instance_double(::Migrations::Database::OfflineConnection) }
before do
allow(::Migrations::Database::OfflineConnection).to receive(:new).and_return(
offline_connection,
)
allow(offline_connection).to receive(:clear!)
allow(step).to receive(:process_item)
allow(offline_connection).to receive(:parametrized_insert_statements).and_return(
[["SQL", [1, 2]], ["SQL", [2, 3]]],
)
end
it "resets stats and clears the offline connection" do
job.run(item)
expect(stats).to have_received(:reset!)
expect(offline_connection).to have_received(:clear!)
end
it "processes an item and logs errors if exceptions occur" do
allow(step).to receive(:process_item).and_raise(StandardError.new("error"))
job.run(item)
expect(stats).to have_received(:log_error).with(
"Failed to process item",
exception: an_instance_of(StandardError),
details: item,
)
end
it "returns the parametrized insert statements and stats" do
result = job.run(item)
expect(result).to eq([[["SQL", [1, 2]], ["SQL", [2, 3]]], stats])
end
end
describe "#cleanup" do
it "can be called without errors" do
expect { job.cleanup }.not_to raise_error
end
end
end

156
migrations/spec/lib/converters/base/progress_stats_spec.rb Normal file
View File

@ -0,0 +1,156 @@
# frozen_string_literal: true
RSpec.describe ::Migrations::Converters::Base::ProgressStats do
subject(:stats) { described_class.new }
describe "#initialize" do
it "starts at the correct values" do
expect(stats.progress).to eq(1)
expect(stats.warning_count).to eq(0)
expect(stats.error_count).to eq(0)
end
end
describe "attribute accessors" do
it "allows reading and writing for :progress" do
stats.progress = 10
expect(stats.progress).to eq(10)
end
it "allows reading and writing for :warning_count" do
stats.warning_count = 5
expect(stats.warning_count).to eq(5)
end
it "allows reading and writing for :error_count" do
stats.error_count = 3
expect(stats.error_count).to eq(3)
end
end
describe "#reset!" do
before do
stats.progress = 5
stats.warning_count = 2
stats.error_count = 3
stats.reset!
end
it "resets progress to 1" do
expect(stats.progress).to eq(1)
end
it "resets warning_count to 0" do
expect(stats.warning_count).to eq(0)
end
it "resets error_count to 0" do
expect(stats.error_count).to eq(0)
end
end
describe "#log_info" do
before { allow(::Migrations::Database::IntermediateDB::LogEntry).to receive(:create!) }
it "logs an info message" do
stats.log_info("Info message")
expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create!).with(
type: ::Migrations::Database::IntermediateDB::LogEntry::INFO,
message: "Info message",
exception: nil,
details: nil,
)
end
it "logs an info message with details" do
stats.log_info("Info message", details: { key: "value" })
expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create!).with(
type: ::Migrations::Database::IntermediateDB::LogEntry::INFO,
message: "Info message",
exception: nil,
details: {
key: "value",
},
)
end
end
describe "#log_warning" do
before { allow(::Migrations::Database::IntermediateDB::LogEntry).to receive(:create!) }
it "logs a warning message and increments warning_count" do
expect { stats.log_warning("Warning message") }.to change { stats.warning_count }.by(1)
expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create!).with(
type: ::Migrations::Database::IntermediateDB::LogEntry::WARNING,
message: "Warning message",
exception: nil,
details: nil,
)
end
it "logs a warning message with exception and details and increments warning_count" do
exception = StandardError.new("Warning exception")
expect {
stats.log_warning("Warning message", exception: exception, details: { key: "value" })
}.to change { stats.warning_count }.by(1)
expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create!).with(
type: ::Migrations::Database::IntermediateDB::LogEntry::WARNING,
message: "Warning message",
exception: exception,
details: {
key: "value",
},
)
end
end
describe "#log_error" do
before { allow(::Migrations::Database::IntermediateDB::LogEntry).to receive(:create!) }
it "logs an error message and increments error_count" do
expect { stats.log_error("Error message") }.to change { stats.error_count }.by(1)
expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create!).with(
type: ::Migrations::Database::IntermediateDB::LogEntry::ERROR,
message: "Error message",
exception: nil,
details: nil,
)
end
it "logs an error message with exception and details and increments error_count" do
exception = StandardError.new("Error exception")
expect {
stats.log_error("Error message", exception: exception, details: { key: "value" })
}.to change { stats.error_count }.by(1)
expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create!).with(
type: ::Migrations::Database::IntermediateDB::LogEntry::ERROR,
message: "Error message",
exception: exception,
details: {
key: "value",
},
)
end
end
describe "#==" do
let(:other_stats) { described_class.new }
it "returns true for objects with the same values" do
expect(stats).to eq(other_stats)
end
it "returns false for objects with different values" do
other_stats.progress = 2
expect(stats).not_to eq(other_stats)
end
end
end

42
migrations/spec/lib/converters/base/serial_job_spec.rb Normal file
View File

@ -0,0 +1,42 @@
# frozen_string_literal: true
RSpec.describe ::Migrations::Converters::Base::SerialJob do
subject(:job) { described_class.new(step) }
let(:step) { instance_double(::Migrations::Converters::Base::ProgressStep) }
let(:item) { "Item" }
let(:stats) do
instance_double(::Migrations::Converters::Base::ProgressStats, reset!: nil, log_error: nil)
end
before { allow(::Migrations::Converters::Base::ProgressStats).to receive(:new).and_return(stats) }
describe "#run" do
it "resets stats and processes item" do
allow(step).to receive(:process_item).and_return(stats)
job.run(item)
expect(stats).to have_received(:reset!)
expect(step).to have_received(:process_item).with(item, stats)
end
it "logs error if processing item raises an exception" do
allow(step).to receive(:process_item).and_raise(StandardError)
job.run(item)
expect(stats).to have_received(:log_error).with(
"Failed to process item",
exception: an_instance_of(StandardError),
details: item,
)
end
end
describe "#cleanup" do
it "can be called without errors" do
expect { job.cleanup }.not_to raise_error
end
end
end

59
migrations/spec/lib/converters/base/step_spec.rb Normal file
View File

@ -0,0 +1,59 @@
# frozen_string_literal: true
RSpec.describe ::Migrations::Converters::Base::Step do
before do
Object.const_set(
"TemporaryModule",
Module.new do
const_set("TopicUsers", Class.new(::Migrations::Converters::Base::Step) {})
const_set("Users", Class.new(::Migrations::Converters::Base::Step) {})
end,
)
end
after do
TemporaryModule.send(:remove_const, "TopicUsers")
TemporaryModule.send(:remove_const, "Users")
Object.send(:remove_const, "TemporaryModule")
end
describe ".title" do
it "uses the classname within title" do
expect(TemporaryModule::TopicUsers.title).to eq("Converting topic users")
expect(TemporaryModule::Users.title).to eq("Converting users")
end
it "uses the `title` attribute if it has been set" do
TemporaryModule::Users.title "Foo bar"
expect(TemporaryModule::Users.title).to eq("Foo bar")
end
end
describe "#initialize" do
it "works when no arguments are supplied" do
step = nil
expect { step = TemporaryModule::Users.new }.not_to raise_error
expect(step.settings).to be_nil
end
it "initializes the `settings` attribute if given" do
settings = { a: 1, b: 2 }
step = TemporaryModule::Users.new(settings:)
expect(step.settings).to eq(settings)
end
it "initializes additional attributes if they exist" do
TemporaryModule::Users.class_eval { attr_accessor :foo, :bar }
settings = { a: 1, b: 2 }
foo = "a string"
bar = false
step = TemporaryModule::Users.new(settings:, foo:, bar:, non_existent: 123)
expect(step.settings).to eq(settings)
expect(step.foo).to eq(foo)
expect(step.bar).to eq(bar)
expect(step).to_not respond_to(:non_existent)
end
end
end

114
migrations/spec/lib/converters/base/worker_spec.rb Normal file
View File

@ -0,0 +1,114 @@
# frozen_string_literal: true
RSpec.describe ::Migrations::Converters::Base::Worker do
subject(:worker) { described_class.new(index, input_queue, output_queue, job) }
let(:index) { 1 }
let(:input_queue) { Queue.new }
let(:output_queue) { Queue.new }
let(:job) do
instance_double(::Migrations::Converters::Base::ParallelJob, run: "result", cleanup: nil)
end
after do
input_queue.close if !input_queue.closed?
output_queue.close if !output_queue.closed?
end
describe "#start" do
it "works when `input_queue` is empty" do
expect do
worker.start
input_queue.close
worker.wait
output_queue.close
end.not_to raise_error
end
it "uses `ForkManager.fork`" do
allow(::Migrations::ForkManager).to receive(:fork).and_call_original
worker.start
input_queue.close
worker.wait
output_queue.close
expect(::Migrations::ForkManager).to have_received(:fork)
end
it "writes the output of `job.run` into `output_queue`" do
allow(job).to receive(:run) { |data| "run: #{data[:text]}" }
worker.start
input_queue << { text: "Item 1" } << { text: "Item 2" } << { text: "Item 3" }
input_queue.close
worker.wait
output_queue.close
expect(output_queue).to have_queue_contents("run: Item 1", "run: Item 2", "run: Item 3")
end
def create_progress_stats(progress: 1, warning_count: 0, error_count: 0)
stats = ::Migrations::Converters::Base::ProgressStats.new
stats.progress = progress
stats.warning_count = warning_count
stats.error_count = error_count
stats
end
it "writes objects to the `output_queue`" do
all_stats = [
create_progress_stats,
create_progress_stats(warning_count: 1),
create_progress_stats(warning_count: 1, error_count: 1),
create_progress_stats(warning_count: 2, error_count: 1),
]
allow(job).to receive(:run) do |data|
index = data[:index]
[index, all_stats[index]]
end
worker.start
input_queue << { index: 0 } << { index: 1 } << { index: 2 } << { index: 3 }
input_queue.close
worker.wait
output_queue.close
expect(output_queue).to have_queue_contents(
[0, all_stats[0]],
[1, all_stats[1]],
[2, all_stats[2]],
[3, all_stats[3]],
)
end
it "runs `job.cleanup` at the end" do
temp_file = Tempfile.new("method_call_check")
temp_file_path = temp_file.path
allow(job).to receive(:run) do |data|
File.write(temp_file_path, "run: #{data[:text]}\n", mode: "a+")
data[:text]
end
allow(job).to receive(:cleanup) do
File.write(temp_file_path, "cleanup\n", mode: "a+")
end
worker.start
input_queue << { text: "Item 1" } << { text: "Item 2" } << { text: "Item 3" }
input_queue.close
worker.wait
output_queue.close
expect(File.read(temp_file_path)).to eq <<~LOG
run: Item 1
run: Item 2
run: Item 3
cleanup
LOG
ensure
temp_file.unlink
end
end
end

105
migrations/spec/lib/converters_spec.rb Normal file
View File

@ -0,0 +1,105 @@
# frozen_string_literal: true
RSpec.describe ::Migrations::Converters do
let(:root_path) { Dir.mktmpdir }
let(:core_path) { File.join(root_path, "lib", "converters") }
let(:private_path) { File.join(root_path, "private", "converters") }
before do
allow(::Migrations).to receive(:root_path).and_return(root_path)
reset_memoization(described_class, :@all_converters)
end
after do
FileUtils.remove_dir(root_path, force: true)
reset_memoization(described_class, :@all_converters)
end
def create_converters(core_names: [], private_names: [])
core_names.each { |dir| FileUtils.mkdir_p(File.join(core_path, dir)) }
private_names.each { |dir| FileUtils.mkdir_p(File.join(private_path, dir)) }
end
describe ".all" do
subject(:all) { described_class.all }
it "returns all the converters except for 'base'" do
create_converters(core_names: %w[base foo bar])
expect(all).to eq(
{ "foo" => File.join(core_path, "foo"), "bar" => File.join(core_path, "bar") },
)
end
it "returns converters from core and private directory" do
create_converters(core_names: %w[base foo bar], private_names: %w[baz qux])
expect(all).to eq(
{
"foo" => File.join(core_path, "foo"),
"bar" => File.join(core_path, "bar"),
"baz" => File.join(private_path, "baz"),
"qux" => File.join(private_path, "qux"),
},
)
end
it "raises an error if there a duplicate names" do
create_converters(core_names: %w[base foo bar], private_names: %w[foo baz qux])
expect { all }.to raise_error(StandardError, /Duplicate converter name found: foo/)
end
end
describe ".names" do
subject(:names) { described_class.names }
it "returns a sorted array of converter names" do
create_converters(core_names: %w[base foo bar], private_names: %w[baz qux])
expect(names).to eq(%w[bar baz foo qux])
end
end
describe ".path_of" do
it "returns the path of a converter" do
create_converters(core_names: %w[base foo bar])
expect(described_class.path_of("foo")).to eq(File.join(core_path, "foo"))
end
it "raises an error if there is no converter" do
create_converters(core_names: %w[base foo bar])
expect { described_class.path_of("baz") }.to raise_error(
StandardError,
"Could not find a converter named 'baz'",
)
expect { described_class.path_of("base") }.to raise_error(
StandardError,
"Could not find a converter named 'base'",
)
end
end
describe ".default_settings_path" do
it "returns the path of the default settings file" do
create_converters(core_names: %w[foo bar])
expect(described_class.default_settings_path("foo")).to eq(
File.join(core_path, "foo", "settings.yml"),
)
expect(described_class.default_settings_path("bar")).to eq(
File.join(core_path, "bar", "settings.yml"),
)
end
it "raises an error if there is no converter" do
create_converters(core_names: %w[foo bar])
expect { described_class.default_settings_path("baz") }.to raise_error(
StandardError,
"Could not find a converter named 'baz'",
)
end
end
end

176
migrations/spec/lib/database/connection_spec.rb Normal file
View File

@ -0,0 +1,176 @@
# frozen_string_literal: true
RSpec.describe ::Migrations::Database::Connection do
def create_connection(**params)
Dir.mktmpdir do |storage_path|
db_path = File.join(storage_path, "test.db")
connection = described_class.new(path: db_path, **params)
return connection if !block_given?
begin
yield connection
ensure
connection.close if connection
end
end
end
describe "class" do
subject(:connection) { create_connection }
after { connection.close }
it_behaves_like "a database connection"
end
describe ".open_database" do
it "creates a database at the given path " do
Dir.mktmpdir do |storage_path|
db_path = File.join(storage_path, "test.db")
db = described_class.open_database(path: db_path)
expect(File.exist?(db_path)).to be true
expect(db.pragma("journal_mode")).to eq("wal")
expect(db.pragma("locking_mode")).to eq("normal")
ensure
db.close if db
end
end
end
describe "#close" do
it "closes the underlying database" do
create_connection do |connection|
db = connection.db
connection.close
expect(db).to be_closed
end
end
it "closes cached prepared statements" do
cache_class = ::Migrations::Database::PreparedStatementCache
cache_double = instance_spy(cache_class)
allow(cache_class).to receive(:new).and_return(cache_double)
create_connection do |connection|
expect(cache_double).not_to have_received(:clear)
connection.close
expect(cache_double).to have_received(:clear).once
end
end
it "commits an active transaction" do
create_connection do |connection|
db = described_class.open_database(path: connection.path)
db.execute("CREATE TABLE foo (id INTEGER)")
connection.insert("INSERT INTO foo (id) VALUES (?)", [1])
connection.insert("INSERT INTO foo (id) VALUES (?)", [2])
expect(db.query_single_splat("SELECT COUNT(*) FROM foo")).to eq(0)
connection.close
expect(db.query_single_splat("SELECT COUNT(*) FROM foo")).to eq(2)
db.close
end
end
end
describe "#closed?" do
it "correctly reports if connection is closed" do
create_connection do |connection|
expect(connection.closed?).to be false
connection.close
expect(connection.closed?).to be true
end
end
end
describe "#insert" do
it "commits inserted rows when reaching `batch_size`" do
transaction_batch_size = 3
create_connection(transaction_batch_size:) do |connection|
db = described_class.open_database(path: connection.path)
db.execute("CREATE TABLE foo (id INTEGER)")
1.upto(10) do |index|
connection.insert("INSERT INTO foo (id) VALUES (?)", [index])
expected_count = index / transaction_batch_size * transaction_batch_size
expect(db.query_single_splat("SELECT COUNT(*) FROM foo")).to eq(expected_count)
end
db.close
end
end
it "works with one and more parameters" do
transaction_batch_size = 1
create_connection(transaction_batch_size:) do |connection|
db = described_class.open_database(path: connection.path)
db.execute("CREATE TABLE foo (id INTEGER)")
db.execute("CREATE TABLE bar (id INTEGER, name TEXT)")
connection.insert("INSERT INTO foo (id) VALUES (?)", [1])
connection.insert("INSERT INTO bar (id, name) VALUES (?, ?)", [1, "Alice"])
expect(db.query_splat("SELECT id FROM foo")).to contain_exactly(1)
expect(db.query("SELECT id, name FROM bar")).to contain_exactly({ id: 1, name: "Alice" })
db.close
end
end
end
context "when `::Migrations::ForkManager.fork` is used" do
it "temporarily closes the connection while a process fork is created" do
create_connection do |connection|
expect(connection.closed?).to be false
connection.db.execute("CREATE TABLE foo (id INTEGER)")
connection.insert("INSERT INTO foo (id) VALUES (?)", [1])
expect(connection.db.query_splat("SELECT id FROM foo")).to contain_exactly(1)
db_before_fork = connection.db
::Migrations::ForkManager.fork do
expect(connection.closed?).to be true
expect(connection.db).to be_nil
end
expect(connection.closed?).to be false
expect(connection.db).to_not eq(db_before_fork)
connection.insert("INSERT INTO foo (id) VALUES (?)", [2])
expect(connection.db.query_splat("SELECT id FROM foo")).to contain_exactly(1, 2)
end
end
it "works with multiple forks" do
create_connection do |connection|
expect(connection.closed?).to be false
::Migrations::ForkManager.fork { expect(connection.closed?).to be true }
expect(connection.closed?).to be false
::Migrations::ForkManager.fork { expect(connection.closed?).to be true }
expect(connection.closed?).to be false
end
end
it "cleans up fork hooks when connection gets closed" do
expect(::Migrations::ForkManager.size).to eq(0)
create_connection do |connection|
expect(::Migrations::ForkManager.size).to eq(2)
connection.close
expect(::Migrations::ForkManager.size).to eq(0)
end
end
end
end

5
migrations/spec/lib/database/intermediate_db/log_entry_spec.rb Normal file
View File

@ -0,0 +1,5 @@
# frozen_string_literal: true
RSpec.describe ::Migrations::Database::IntermediateDB::LogEntry do
it_behaves_like "a database entity"
end

97
migrations/spec/lib/database/intermediate_db_spec.rb Normal file
View File

@ -0,0 +1,97 @@
# frozen_string_literal: true
RSpec.describe ::Migrations::Database::IntermediateDB do
before { reset_memoization(described_class, :@db) }
after { reset_memoization(described_class, :@db) }
def create_connection_double
connection = instance_double(::Migrations::Database::Connection)
allow(connection).to receive(:insert)
allow(connection).to receive(:close)
connection
end
describe ".setup" do
it "works with `::Migrations::Database::Connection`" do
Dir.mktmpdir do |storage_path|
db_path = File.join(storage_path, "test.db")
connection = ::Migrations::Database::Connection.new(path: db_path)
connection.db.execute("CREATE TABLE foo (id INTEGER)")
described_class.setup(connection)
described_class.insert("INSERT INTO foo (id) VALUES (?)", 1)
described_class.insert("INSERT INTO foo (id) VALUES (?)", 2)
expect(connection.db.query_splat("SELECT id FROM foo")).to contain_exactly(1, 2)
connection.close
end
end
it "works with `::Migrations::Database::OfflineConnection`" do
connection = ::Migrations::Database::OfflineConnection.new
described_class.setup(connection)
described_class.insert("INSERT INTO foo (id, name) VALUES (?, ?)", 1, "Alice")
described_class.insert("INSERT INTO foo (id, name) VALUES (?, ?)", 2, "Bob")
expect(connection.parametrized_insert_statements).to eq(
[
["INSERT INTO foo (id, name) VALUES (?, ?)", [1, "Alice"]],
["INSERT INTO foo (id, name) VALUES (?, ?)", [2, "Bob"]],
],
)
connection.close
end
it "switches the connection" do
old_connection = create_connection_double
new_connection = create_connection_double
sql = "INSERT INTO foo (id) VALUES (?)"
described_class.setup(old_connection)
described_class.insert(sql, 1)
expect(old_connection).to have_received(:insert).with(sql, [1])
expect(new_connection).to_not have_received(:insert)
described_class.setup(new_connection)
described_class.insert(sql, 2)
expect(old_connection).to_not have_received(:insert).with(sql, [2])
expect(new_connection).to have_received(:insert).with(sql, [2])
end
it "closes a previous connection" do
old_connection = create_connection_double
new_connection = create_connection_double
described_class.setup(old_connection)
described_class.setup(new_connection)
expect(old_connection).to have_received(:close)
expect(new_connection).to_not have_received(:close)
end
end
context "with fake connection" do
let(:connection) { create_connection_double }
let!(:sql) { "INSERT INTO foo (id, name) VALUES (?, ?)" }
before { described_class.setup(connection) }
describe ".insert" do
it "calls `#insert` on the connection" do
described_class.insert(sql, 1, "Alice")
expect(connection).to have_received(:insert).with(sql, [1, "Alice"])
end
end
describe ".close" do
it "closes the underlying connection" do
described_class.close
expect(connection).to have_received(:close).with(no_args)
end
end
end
end

104
migrations/spec/lib/database/migrator_spec.rb Normal file
View File

@ -0,0 +1,104 @@
# frozen_string_literal: true
RSpec.describe ::Migrations::Database::Migrator do
def migrate(
migrations_directory: nil,
migrations_path: nil,
storage_path: nil,
db_filename: "intermediate.db",
ignore_errors: false
)
if migrations_directory
migrations_path =
File.join(
::Migrations.root_path,
"spec",
"support",
"fixtures",
"schema",
migrations_directory,
)
end
temp_path = storage_path = Dir.mktmpdir if storage_path.nil?
db_path = File.join(storage_path, db_filename)
begin
described_class.new(db_path).migrate(migrations_path)
rescue StandardError
raise unless ignore_errors
end
yield db_path, storage_path
ensure
FileUtils.remove_dir(temp_path, force: true) if temp_path
end
describe "#migrate" do
it "works with the IntermediateDB schema" do
migrate(
migrations_path: ::Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH,
db_filename: "intermediate.db",
) do |db_path, storage_path|
expect(Dir.children(storage_path)).to contain_exactly("intermediate.db")
db = Extralite::Database.new(db_path)
expect(db.tables).not_to be_empty
db.close
end
end
it "works with the UploadsDB schema" do
migrate(
migrations_path: ::Migrations::Database::UPLOADS_DB_SCHEMA_PATH,
db_filename: "uploads.db",
) do |db_path, storage_path|
expect(Dir.children(storage_path)).to contain_exactly("uploads.db")
db = Extralite::Database.new(db_path)
expect(db.tables).not_to be_empty
db.close
end
end
it "executes schema files" do
Dir.mktmpdir do |storage_path|
migrate(migrations_directory: "one", storage_path:) do |db_path|
db = Extralite::Database.new(db_path)
expect(db.tables).to contain_exactly("first_table", "schema_migrations")
db.close
end
migrate(migrations_directory: "one", storage_path:) do |db_path|
db = Extralite::Database.new(db_path)
expect(db.tables).to contain_exactly("first_table", "schema_migrations")
db.close
end
migrate(migrations_directory: "two", storage_path:) do |db_path|
db = Extralite::Database.new(db_path)
expect(db.tables).to contain_exactly("first_table", "second_table", "schema_migrations")
db.close
end
end
end
end
describe "#reset!" do
it "deletes all DB related files" do
migrate(migrations_directory: "invalid", ignore_errors: true) do |db_path, storage_path|
File.write(File.join(storage_path, "hello_world.txt"), "Hello World!")
expect(Dir.children(storage_path)).to contain_exactly(
"intermediate.db",
"intermediate.db-shm",
"intermediate.db-wal",
"hello_world.txt",
)
described_class.new(db_path).reset!
expect(Dir.children(storage_path)).to contain_exactly("hello_world.txt")
end
end
end
end

67
migrations/spec/lib/database/offline_connection_spec.rb Normal file
View File

@ -0,0 +1,67 @@
# frozen_string_literal: true
RSpec.describe ::Migrations::Database::OfflineConnection do
subject(:connection) { described_class.new }
let!(:sql) { "INSERT INTO foo (id, name) VALUES (?, ?)" }
it_behaves_like "a database connection"
describe "#close" do
it "removes the cached statements" do
connection.insert(sql, [1, "Alice"])
connection.insert(sql, [2, "Bob"])
expect(connection.parametrized_insert_statements).to_not be_empty
connection.close
expect(connection.parametrized_insert_statements).to be_nil
end
end
describe "#closed?" do
it "correctly reports if connection is closed" do
expect(connection.closed?).to be false
connection.close
expect(connection.closed?).to be true
end
end
describe "#insert" do
it "can be called without errors" do
expect { connection.insert(sql, [1, "Alice"]) }.not_to raise_error
end
end
describe "#parametrized_insert_statements" do
it "returns an empty array if nothing has been cached" do
expect(connection.parametrized_insert_statements).to eq([])
end
it "returns the cached INSERT statements and parameters in original order" do
connection.insert(sql, [1, "Alice"])
connection.insert(sql, [2, "Bob"])
connection.insert(sql, [3, "Carol"])
expected_data = [[sql, [1, "Alice"]], [sql, [2, "Bob"]], [sql, [3, "Carol"]]]
expect(connection.parametrized_insert_statements).to eq(expected_data)
# multiple calls return the same data
expect(connection.parametrized_insert_statements).to eq(expected_data)
expect(connection.parametrized_insert_statements).to eq(expected_data)
end
end
describe "#clear!" do
it "clears all cached data" do
connection.insert(sql, [1, "Alice"])
connection.insert(sql, [2, "Bob"])
connection.insert(sql, [3, "Carol"])
expect(connection.parametrized_insert_statements).to_not be_empty
connection.clear!
expect(connection.parametrized_insert_statements).to eq([])
end
end
end

41
migrations/spec/lib/database/prepared_statement_cache_spec.rb Normal file
View File

@ -0,0 +1,41 @@
# frozen_string_literal: true
require "extralite"
RSpec.describe ::Migrations::Database::PreparedStatementCache do
let(:cache) { described_class.new(3) }
def create_statement_double
instance_double(Extralite::Query, close: nil)
end
it "should inherit behavior from LruRedux::Cache" do
expect(described_class).to be < LruRedux::Cache
end
it "closes the statement when an old entry is removed" do
cache["a"] = a_statement = create_statement_double
cache["b"] = b_statement = create_statement_double
cache["c"] = c_statement = create_statement_double
# this should remove the oldest entry "a" from the cache and call #close on the statement
cache["d"] = d_statement = create_statement_double
expect(a_statement).to have_received(:close)
expect(b_statement).not_to have_received(:close)
expect(c_statement).not_to have_received(:close)
expect(d_statement).not_to have_received(:close)
end
it "closes all statements when the cache is cleared" do
cache["a"] = a_statement = create_statement_double
cache["b"] = b_statement = create_statement_double
cache["c"] = c_statement = create_statement_double
cache.clear
expect(a_statement).to have_received(:close)
expect(b_statement).to have_received(:close)
expect(c_statement).to have_received(:close)
end
end

143
migrations/spec/lib/database_spec.rb Normal file
View File

@ -0,0 +1,143 @@
# frozen_string_literal: true
RSpec.describe ::Migrations::Database do
context "with `Migrator`" do
let(:db_path) { "path/to/db" }
let(:migrations_path) { "path/to/migrations" }
let(:migrator_instance) { instance_double(::Migrations::Database::Migrator) }
before do
allow(::Migrations::Database::Migrator).to receive(:new).with(db_path).and_return(
migrator_instance,
)
allow(::Migrations::Database::Migrator).to receive(:new).with(db_path).and_return(
migrator_instance,
)
end
describe ".migrate" do
it "migrates the database" do
allow(migrator_instance).to receive(:migrate)
described_class.migrate(db_path, migrations_path:)
expect(::Migrations::Database::Migrator).to have_received(:new).with(db_path)
expect(migrator_instance).to have_received(:migrate).with(migrations_path)
end
end
describe ".reset!" do
it "resets the database" do
allow(migrator_instance).to receive(:reset!)
described_class.reset!(db_path)
expect(::Migrations::Database::Migrator).to have_received(:new).with(db_path)
expect(migrator_instance).to have_received(:reset!)
end
end
end
describe ".connect" do
it "yields a new connection and closes it after the block" do
Dir.mktmpdir do |storage_path|
db_path = File.join(storage_path, "test.db")
db = nil
described_class.connect(db_path) do |connection|
expect(connection).to be_a(::Migrations::Database::Connection)
expect(connection.path).to eq(db_path)
db = connection.db
expect(db).not_to be_closed
end
expect(db).to be_closed
end
end
it "closes the connection even if an exception is raised within block" do
Dir.mktmpdir do |storage_path|
db_path = File.join(storage_path, "test.db")
db = nil
expect {
described_class.connect(db_path) do |connection|
db = connection.db
expect(db).not_to be_closed
raise "boom"
end
}.to raise_error(StandardError)
expect(db).to be_closed
end
end
end
describe ".format_datetime" do
it "formats a DateTime object to ISO 8601 string" do
datetime = DateTime.new(2023, 10, 5, 17, 30, 0)
expect(described_class.format_datetime(datetime)).to eq("2023-10-05T17:30:00Z")
end
it "returns nil for nil input" do
expect(described_class.format_datetime(nil)).to be_nil
end
end
describe ".format_date" do
it "formats a Date object to ISO 8601 string" do
date = Date.new(2023, 10, 5)
expect(described_class.format_date(date)).to eq("2023-10-05")
end
it "returns nil for nil input" do
expect(described_class.format_date(nil)).to be_nil
end
end
describe ".format_boolean" do
it "returns 1 for true" do
expect(described_class.format_boolean(true)).to eq(1)
end
it "returns 0 for false" do
expect(described_class.format_boolean(false)).to eq(0)
end
it "returns nil for nil input" do
expect(described_class.format_boolean(nil)).to be_nil
end
end
describe ".format_ip_address" do
it "formats a valid IPv4 address" do
expect(described_class.format_ip_address("192.168.1.1")).to eq("192.168.1.1")
end
it "formats a valid IPv6 address" do
expect(described_class.format_ip_address("2001:0db8:85a3:0000:0000:8a2e:0370:7334")).to eq(
"2001:db8:85a3::8a2e:370:7334",
)
end
it "returns nil for an invalid IP address" do
expect(described_class.format_ip_address("invalid_ip")).to be_nil
end
it "returns nil for nil input" do
expect(described_class.format_ip_address(nil)).to be_nil
end
end
describe ".to_blob" do
it "converts a string to a `Extralite::Blob`" do
expect(described_class.to_blob("Hello, 世界!")).to be_a(Extralite::Blob)
end
it "returns nil for nil input" do
expect(described_class.to_blob(nil)).to be_nil
end
end
end

36
migrations/spec/lib/migrations_spec.rb
View File

@ -1,43 +1,9 @@
# frozen_string_literal: true
require_relative "../../lib/migrations"
RSpec.describe Migrations do
RSpec.describe ::Migrations do
describe ".root_path" do
it "returns the root path" do
expect(described_class.root_path).to eq(File.expand_path("../..", __dir__))
end
end
describe ".load_gemfiles" do
it "exits with error if the gemfile does not exist" do
relative_path = "does_not_exist"
expect { described_class.load_gemfiles(relative_path) }.to output(
include("Could not find Gemfile").and include(relative_path)
).to_stderr.and raise_error(SystemExit) { |error| expect(error.status).to eq(1) }
end
def with_temporary_root_path
Dir.mktmpdir do |temp_dir|
described_class.stubs(:root_path).returns(temp_dir)
yield temp_dir
end
end
it "exits with an error if the required Ruby version isn't found" do
with_temporary_root_path do |root_path|
gemfile_path = File.join(root_path, "config/gemfiles/test/Gemfile")
FileUtils.mkdir_p(File.dirname(gemfile_path))
File.write(gemfile_path, <<~GEMFILE)
source "http://localhost"
ruby "~> 100.0.0"
GEMFILE
expect { described_class.load_gemfiles("test") }.to output(
include("your Gemfile specified ~> 100.0.0"),
).to_stderr.and raise_error(SystemExit) { |error| expect(error.status).to eq(1) }
end
end
end
end

21
migrations/spec/rails_helper.rb
View File

@ -3,18 +3,13 @@
# we need to require the rails_helper from core to load the Rails environment
require_relative "../../spec/rails_helper"
require "bundler/inline"
require "bundler/ui"
require_relative "../lib/migrations"
# this is a hack to allow us to load Gemfiles for converters
Dir[File.expand_path("../config/gemfiles/**/Gemfile", __dir__)].each do |path|
# Create new UI and set level to confirm to avoid printing unnecessary messages
bundler_ui = Bundler::UI::Shell.new
bundler_ui.level = "confirm"
::Migrations.configure_zeitwerk
::Migrations.enable_i18n
gemfile(true, ui: bundler_ui) do
# rubocop:disable Security/Eval
eval(File.read(path), nil, path, 1)
# rubocop:enable Security/Eval
end
end
require "rspec-multi-mock"
Dir[File.expand_path("./support/**/*.rb", __dir__)].each { |f| require f }
RSpec.configure { |config| config.mock_with MultiMock::Adapter.for(:rspec, :mocha) }

17
migrations/spec/support/fixtures/schema/copy/schema.sql Normal file
View File

@ -0,0 +1,17 @@
CREATE TABLE users
(
id INTEGER NOT NULL PRIMARY KEY,
username TEXT NOT NULL UNIQUE
);
CREATE TABLE config
(
name TEXT NOT NULL PRIMARY KEY,
value TEXT NOT NULL
);
CREATE TABLE uploads
(
id INTEGER NOT NULL PRIMARY KEY,
url TEXT NOT NULL
);

1
migrations/spec/support/fixtures/schema/invalid/001-invalid.sql Normal file
View File

@ -0,0 +1 @@
CREATE TABLE defect

4
migrations/spec/support/fixtures/schema/one/001-first-table.sql Normal file
View File

@ -0,0 +1,4 @@
CREATE TABLE first_table
(
id INTEGER TEXT NOT NULL PRIMARY KEY
);

4
migrations/spec/support/fixtures/schema/two/001-first-table.sql Normal file
View File

@ -0,0 +1,4 @@
CREATE TABLE first_table
(
id INTEGER TEXT NOT NULL PRIMARY KEY
);

4
migrations/spec/support/fixtures/schema/two/002-second-table.sql Normal file
View File

@ -0,0 +1,4 @@
CREATE TABLE second_table
(
id INTEGER TEXT NOT NULL PRIMARY KEY
);

7
migrations/spec/support/helpers.rb Normal file
View File

@ -0,0 +1,7 @@
# frozen_string_literal: true
def reset_memoization(instance, *variables)
variables.each do |var|
instance.remove_instance_variable(var) if instance.instance_variable_defined?(var)
end
end

11
migrations/spec/support/matchers/have_constant.rb Normal file
View File

@ -0,0 +1,11 @@
# frozen_string_literal: true
RSpec::Matchers.define :have_constant do |const|
match { |owner| owner.const_defined?(const) }
failure_message { |owner| "expected #{owner} to have a constant #{const}" }
failure_message_when_negated do |owner|
"expected #{owner} not to have a constant #{const}, but it does"
end
end

19
migrations/spec/support/matchers/have_queue_contents.rb Normal file
View File

@ -0,0 +1,19 @@
# frozen_string_literal: true
RSpec::Matchers.define :have_queue_contents do |*expected|
match do |queue|
@actual = []
@actual << queue.pop(true) until queue.empty?
@actual == expected
rescue ThreadError
@actual == expected
end
failure_message do
"expected queue to have contents #{expected.inspect}, but got #{@actual.inspect}"
end
failure_message_when_negated do
"expected queue not to have contents #{expected.inspect}, but it did"
end
end

15
migrations/spec/support/shared_examples/database_connection.rb Normal file
View File

@ -0,0 +1,15 @@
# frozen_string_literal: true
RSpec.shared_examples "a database connection" do
it "responds to #insert" do
expect(subject).to respond_to(:insert).with(1..2).arguments
end
it "responds to #close" do
expect(subject).to respond_to(:close).with(0).arguments
end
it "responds to #closed?" do
expect(subject).to respond_to(:closed?).with(0).arguments
end
end

11
migrations/spec/support/shared_examples/database_entity.rb Normal file
View File

@ -0,0 +1,11 @@
# frozen_string_literal: true
RSpec.shared_examples "a database entity" do
it "has SQL constant" do
expect(subject).to have_constant(:SQL)
end
it "responds to .create!" do
expect(subject).to respond_to(:create!)
end
end