From 6580025af93d0df129d7bc012a7ca2afba81756c Mon Sep 17 00:00:00 2001
From: Sam Saffron <sam.saffron@gmail.com>
Date: Mon, 27 May 2019 16:51:24 +1000
Subject: [PATCH] FEATURE: add backup directory for mmdb files

This new `DISCOURSE_MAXMIND_BACKUP_PATH` can be used a secondary location
for maxmind db. That way a build machine, for example can cache it on the
host and reuse between builds.

Also per 5bfeef77 added proper error raising for download fails from
dedicated rake task

This also moves "refresh_maxmind_db_during_precompile_days" to a global
setting, it did not make sense in a site setting
---
 config/discourse_defaults.conf |  7 ++++
 config/site_settings.yml       |  6 ---
 lib/discourse_ip_info.rb       | 20 ++++------
 lib/tasks/assets.rake          | 70 +++++++++++++++++++++++++++++++---
 4 files changed, 80 insertions(+), 23 deletions(-)

diff --git a/config/discourse_defaults.conf b/config/discourse_defaults.conf
index 784d395d369..60bb0ec10f4 100644
--- a/config/discourse_defaults.conf
+++ b/config/discourse_defaults.conf
@@ -208,3 +208,10 @@ max_old_rebakes_per_15_minutes = 300
 # maximum number of log messages in /logs
 max_logster_logs = 1000
 
+# during precompile update maxmind database if older than N days
+# set to 0 to disable
+refresh_maxmind_db_during_precompile_days = 2
+
+# backup path containing maxmind db files
+maxmind_backup_path =
+
diff --git a/config/site_settings.yml b/config/site_settings.yml
index a1b4a089b77..aefa536c409 100644
--- a/config/site_settings.yml
+++ b/config/site_settings.yml
@@ -1534,12 +1534,6 @@ developer:
   enable_safe_mode:
     default: true
     client: true
-  refresh_maxmind_db_during_precompile_days:
-    min: 0
-    max: 120
-    default: 2
-    hidden: true
-    shadowed_by_global: true
 
 embedding:
   feed_polling_enabled:
diff --git a/lib/discourse_ip_info.rb b/lib/discourse_ip_info.rb
index 0af821e5864..37b0fa2e74d 100644
--- a/lib/discourse_ip_info.rb
+++ b/lib/discourse_ip_info.rb
@@ -27,20 +27,16 @@ class DiscourseIpInfo
   def self.mmdb_download(name)
     FileUtils.mkdir_p(path)
 
-    begin
-      gz_file = FileHelper.download(
-        "https://geolite.maxmind.com/geoip/databases/#{name}/update",
-        max_file_size: 100.megabytes,
-        tmp_file_name: "#{name}.gz"
-      )
+    gz_file = FileHelper.download(
+      "https://geolite.maxmind.com/geoip/databases/#{name}/update",
+      max_file_size: 100.megabytes,
+      tmp_file_name: "#{name}.gz"
+    )
 
-      Discourse::Utils.execute_command("gunzip", gz_file.path)
+    Discourse::Utils.execute_command("gunzip", gz_file.path)
 
-      path = gz_file.path.sub(/\.gz\z/, "")
-      FileUtils.mv(path, mmdb_path(name))
-    rescue OpenURI::HTTPError => e
-      Rails.logger.warn("MaxMindDB (#{name}) could not be downloaded: #{e}")
-    end
+    path = gz_file.path.sub(/\.gz\z/, "")
+    FileUtils.mv(path, mmdb_path(name))
   ensure
     gz_file&.close!
   end
diff --git a/lib/tasks/assets.rake b/lib/tasks/assets.rake
index 9fe3a485d60..dca6e6dba37 100644
--- a/lib/tasks/assets.rake
+++ b/lib/tasks/assets.rake
@@ -158,15 +158,75 @@ def concurrent?
   end
 end
 
+def get_mmdb_time(root_path)
+  mmdb_time = nil
+
+  %w{
+    GeoLite2-City
+    GeoLite2-ASN
+  }.map do |name|
+
+    path = File.join(root_path, "#{name}.mmdb")
+
+    if File.exist?(path)
+      mmdb_time = File.mtime(path)
+    else
+      mmdb_time = nil
+      break
+    end
+  end
+
+  mmdb_time
+end
+
+def copy_maxmind(from_path, to_path)
+  puts "Copying MaxMindDB from #{from_path} to #{to_path}"
+
+  %w{
+    GeoLite2-City
+    GeoLite2-ASN
+  }.each do |name|
+    from = File.join(from_path, "#{name}.mmdb")
+    to = File.join(to_path, "#{name}.mmdb")
+    FileUtils.cp(from, to, preserve: true)
+  end
+end
+
 task 'assets:precompile' => 'assets:precompile:before' do
-  if refresh_days = SiteSetting.refresh_maxmind_db_during_precompile_days
-    mmdb_path = DiscourseIpInfo.mmdb_path('GeoLite2-City')
-    mmdb_time = File.exist?(mmdb_path) && File.mtime(mmdb_path)
+
+  refresh_days = GlobalSetting.refresh_maxmind_db_during_precompile_days
+
+  if refresh_days.to_i > 0
+
+    mmdb_time = get_mmdb_time(DiscourseIpInfo.path)
+
+    backup_mmdb_time =
+      if GlobalSetting.maxmind_backup_path.present?
+        get_mmdb_time(GlobalSetting.maxmind_backup_path)
+      end
+
+    mmdb_time ||= backup_mmdb_time
+    if backup_mmdb_time && backup_mmdb_time >= mmdb_time
+      copy_maxmind(GlobalSetting.maxmind_backup_path, DiscourseIpInfo.path)
+    end
+
     if !mmdb_time || mmdb_time < refresh_days.days.ago
       puts "Downloading MaxMindDB..."
       mmdb_thread = Thread.new do
-        DiscourseIpInfo.mmdb_download('GeoLite2-City')
-        DiscourseIpInfo.mmdb_download('GeoLite2-ASN')
+        begin
+          DiscourseIpInfo.mmdb_download('GeoLite2-City')
+          DiscourseIpInfo.mmdb_download('GeoLite2-ASN')
+
+          if GlobalSetting.maxmind_backup_path.present?
+            copy_maxmind(DiscourseIpInfo.path, GlobalSetting.maxmind_backup_path)
+          end
+
+        rescue OpenURI::HTTPError => e
+          STDERR.puts("*" * 100)
+          STDERR.puts("MaxMindDB (#{name}) could not be downloaded: #{e}")
+          STDERR.puts("*" * 100)
+          Rails.logger.warn("MaxMindDB (#{name}) could not be downloaded: #{e}")
+        end
       end
     end
   end