diff --git a/config/discourse_defaults.conf b/config/discourse_defaults.conf index 784d395d369..60bb0ec10f4 100644 --- a/config/discourse_defaults.conf +++ b/config/discourse_defaults.conf @@ -208,3 +208,10 @@ max_old_rebakes_per_15_minutes = 300 # maximum number of log messages in /logs max_logster_logs = 1000 +# during precompile update maxmind database if older than N days +# set to 0 to disable +refresh_maxmind_db_during_precompile_days = 2 + +# backup path containing maxmind db files +maxmind_backup_path = + diff --git a/config/site_settings.yml b/config/site_settings.yml index a1b4a089b77..aefa536c409 100644 --- a/config/site_settings.yml +++ b/config/site_settings.yml @@ -1534,12 +1534,6 @@ developer: enable_safe_mode: default: true client: true - refresh_maxmind_db_during_precompile_days: - min: 0 - max: 120 - default: 2 - hidden: true - shadowed_by_global: true embedding: feed_polling_enabled: diff --git a/lib/discourse_ip_info.rb b/lib/discourse_ip_info.rb index 0af821e5864..37b0fa2e74d 100644 --- a/lib/discourse_ip_info.rb +++ b/lib/discourse_ip_info.rb @@ -27,20 +27,16 @@ class DiscourseIpInfo def self.mmdb_download(name) FileUtils.mkdir_p(path) - begin - gz_file = FileHelper.download( - "https://geolite.maxmind.com/geoip/databases/#{name}/update", - max_file_size: 100.megabytes, - tmp_file_name: "#{name}.gz" - ) + gz_file = FileHelper.download( + "https://geolite.maxmind.com/geoip/databases/#{name}/update", + max_file_size: 100.megabytes, + tmp_file_name: "#{name}.gz" + ) - Discourse::Utils.execute_command("gunzip", gz_file.path) + Discourse::Utils.execute_command("gunzip", gz_file.path) - path = gz_file.path.sub(/\.gz\z/, "") - FileUtils.mv(path, mmdb_path(name)) - rescue OpenURI::HTTPError => e - Rails.logger.warn("MaxMindDB (#{name}) could not be downloaded: #{e}") - end + path = gz_file.path.sub(/\.gz\z/, "") + FileUtils.mv(path, mmdb_path(name)) ensure gz_file&.close! end diff --git a/lib/tasks/assets.rake b/lib/tasks/assets.rake index 9fe3a485d60..dca6e6dba37 100644 --- a/lib/tasks/assets.rake +++ b/lib/tasks/assets.rake @@ -158,15 +158,75 @@ def concurrent? end end +def get_mmdb_time(root_path) + mmdb_time = nil + + %w{ + GeoLite2-City + GeoLite2-ASN + }.map do |name| + + path = File.join(root_path, "#{name}.mmdb") + + if File.exist?(path) + mmdb_time = File.mtime(path) + else + mmdb_time = nil + break + end + end + + mmdb_time +end + +def copy_maxmind(from_path, to_path) + puts "Copying MaxMindDB from #{from_path} to #{to_path}" + + %w{ + GeoLite2-City + GeoLite2-ASN + }.each do |name| + from = File.join(from_path, "#{name}.mmdb") + to = File.join(to_path, "#{name}.mmdb") + FileUtils.cp(from, to, preserve: true) + end +end + task 'assets:precompile' => 'assets:precompile:before' do - if refresh_days = SiteSetting.refresh_maxmind_db_during_precompile_days - mmdb_path = DiscourseIpInfo.mmdb_path('GeoLite2-City') - mmdb_time = File.exist?(mmdb_path) && File.mtime(mmdb_path) + + refresh_days = GlobalSetting.refresh_maxmind_db_during_precompile_days + + if refresh_days.to_i > 0 + + mmdb_time = get_mmdb_time(DiscourseIpInfo.path) + + backup_mmdb_time = + if GlobalSetting.maxmind_backup_path.present? + get_mmdb_time(GlobalSetting.maxmind_backup_path) + end + + mmdb_time ||= backup_mmdb_time + if backup_mmdb_time && backup_mmdb_time >= mmdb_time + copy_maxmind(GlobalSetting.maxmind_backup_path, DiscourseIpInfo.path) + end + if !mmdb_time || mmdb_time < refresh_days.days.ago puts "Downloading MaxMindDB..." mmdb_thread = Thread.new do - DiscourseIpInfo.mmdb_download('GeoLite2-City') - DiscourseIpInfo.mmdb_download('GeoLite2-ASN') + begin + DiscourseIpInfo.mmdb_download('GeoLite2-City') + DiscourseIpInfo.mmdb_download('GeoLite2-ASN') + + if GlobalSetting.maxmind_backup_path.present? + copy_maxmind(DiscourseIpInfo.path, GlobalSetting.maxmind_backup_path) + end + + rescue OpenURI::HTTPError => e + STDERR.puts("*" * 100) + STDERR.puts("MaxMindDB (#{name}) could not be downloaded: #{e}") + STDERR.puts("*" * 100) + Rails.logger.warn("MaxMindDB (#{name}) could not be downloaded: #{e}") + end end end end