new 'uploads:migrate_to_new_pattern' task

lib/tasks/uploads.rake

@@ -369,6 +369,9 @@ end
 ################################################################################
 
 task "uploads:migrate_to_new_pattern" => :environment do
+  require "file_helper"
+  require "file_store/local_store"
+
   ENV["RAILS_DB"] ? migrate_to_new_pattern : migrate_to_new_pattern_all_sites
 end
 
@@ -389,44 +392,59 @@ def migrate_to_new_pattern
 end
 
 def migrate_uploads_to_new_pattern
-  if Upload.where(sha1: nil).exists?
-    puts "Computing missing SHAs..."
-
-    Upload.where(sha1: nil).find_each do |upload|
-      path = Discourse.store.path_for(upload)
-      size = File.size(path) rescue 0
-      if size > 0
-        upload.sha1 = Digest::SHA1.file(path).hexdigest
-        upload.save
-        putc "."
-      else
-        upload.destroy
-        putc "X"
-      end
-    end
-
-    puts
-  end
-
   puts "Moving uploads to new location..."
-  Upload.where.not(sha1: nil)
+
-        .where("url LIKE '/uploads/%'")
+  max_file_size_kb = [SiteSetting.max_image_size_kb, SiteSetting.max_attachment_size_kb].max.kilobytes
-        .where("url NOT LIKE '/uploads/%/original/%'")
+  local_store = FileStore::LocalStore.new
-        .find_each do |upload|
+
-    path = Discourse.store.path_for(upload)
+  Upload.where("LENGTH(COALESCE(url, '')) = 0").destroy_all
-    if File.exists?(path)
+
-      file = File.open(path)
+  Upload.where("url NOT LIKE '%/original/_X/%'").find_each do |upload|
-      # copy file to new location
+    begin
-      url = Discourse.store.store_upload(file, upload)
+      successful = false
-      file.try(:close!) rescue nil
+      # keep track of the url
-      # remap URLs
+      previous_url = upload.url.dup
-      remap(upload.url, url)
+      # where is the file currently stored?
-      # remove old file
+      external = previous_url =~ /^\/\//
-      FileUtils.rm(path, force: true) rescue nil
+      # download if external
-      putc "."
+      if external
-    else
+        url = SiteSetting.scheme + ":" + previous_url
-      # upload.destroy
+        file = FileHelper.download(url, max_file_size_kb, "discourse", true) rescue nil
-      putc "X"
+        next unless file
+        path = file.path
+      else
+        path = local_store.path_for(upload)
+        next unless File.exists?(path)
+      end
+      # compute SHA if missing
+      if upload.sha1.blank?
+        upload.sha1 = Digest::SHA1.file(path).hexdigest
+      end
+      # optimize if image
+      if FileHelper.is_image?(File.basename(path))
+        ImageOptim.new.optimize_image!(path)
+      end
+      # store to new location & update the filesize
+      File.open(path) do |f|
+        upload.url = Discourse.store.store_upload(f, upload)
+        upload.filesize = f.size
+        upload.save
+      end
+      # remap the URLs
+      remap(previous_url, upload.url)
+      # remove the old file (when local)
+      unless external
+        FileUtils.rm(path, force: true) rescue nil
+      end
+      # succesfully migrated
+      successful = true
+    rescue => e
+      puts e.message
+      puts e.backtrace.join("\n")
+    ensure
+      putc successful ? '.' : 'X'
+      file.try(:unlink) rescue nil
+      file.try(:close) rescue nil
     end
   end
 
@@ -434,45 +452,55 @@ def migrate_uploads_to_new_pattern
 end
 
 def migrate_optimized_images_to_new_pattern
-  if OptimizedImage.where(sha1: nil).exists?
+  max_file_size_kb = SiteSetting.max_image_size_kb.kilobytes
-    puts "Computing missing SHAs..."
+  local_store = FileStore::LocalStore.new
 
-    OptimizedImage.where(sha1: nil).find_each do |optimized_image|
+  OptimizedImage.where("LENGTH(COALESCE(url, '')) = 0").destroy_all
-      path = Discourse.store.path_for(optimized_image)
+
-      size = File.size(path) rescue 0
+  OptimizedImage.where("url NOT LIKE '%/original/_X/%'").find_each do |optimized_image|
-      if size > 0
+    begin
-        optimized_image.sha1 = Digest::SHA1.file(path).hexdigest
+      successful = false
-        optimized_image.save
+      # keep track of the url
-        putc "."
+      previous_url = optimized_image.url.dup
+      # where is the file currently stored?
+      external = previous_url =~ /^\/\//
+      # download if external
+      if external
+        url = SiteSetting.scheme + ":" + previous_url
+        file = FileHelper.download(url, max_file_size_kb, "discourse", true) rescue nil
+        next unless file
+        path = file.path
       else
-        optimized_image.destroy
+        path = local_store.path_for(optimized_image)
-        putc "X"
+        next unless File.exists?(path)
+        file = File.open(path)
       end
-    end
+      # compute SHA if missing
-
+      if optimized_image.sha1.blank?
-    puts
+        optimized_image.sha1 = Digest::SHA1.file(path).hexdigest
-  end
+      end
-
+      # optimize if image
-  puts "Moving optimized images to new location..."
+      ImageOptim.new.optimize_image!(path)
-  OptimizedImage.where.not(sha1: nil)
+      # store to new location & update the filesize
-                .where("width > 0 AND height > 0")
+      File.open(path) do |f|
-                .where("url LIKE '/uploads/%/_optimized/%'")
+        optimized_image.url = Discourse.store.store_optimized_image(f, optimized_image)
-                .where("url NOT LIKE '/uploads/%/optimized/%'")
+        optimized_image.save
-                .find_each do |optimized_image|
+      end
-    path = Discourse.store.path_for(optimized_image)
+      # remap the URLs
-    if File.exists?(path)
+      remap(previous_url, optimized_image.url)
-      file = File.open(path)
+      # remove the old file (when local)
-      # copy file to new location
+      unless external
-      url = Discourse.store.store_optimized_image(file, optimized_image)
+        FileUtils.rm(path, force: true) rescue nil
-      file.try(:close!) rescue nil
+      end
-      # remap URLs
+      # succesfully migrated
-      remap(optimized_image.url, url)
+      successful = true
-      # remove old file
+    rescue => e
-      FileUtils.rm(path, force: true) rescue nil
+      puts e.message
-      putc "."
+      puts e.backtrace.join("\n")
-    else
+    ensure
-      optimized_image.destroy
+      putc successful ? '.' : 'X'
-      putc "X"
+      file.try(:unlink) rescue nil
+      file.try(:close) rescue nil
     end
   end