mirror of
https://github.com/discourse/discourse.git
synced 2024-11-28 12:55:49 +08:00
76c86a4269
The previous fix in e83d35d6 was incorrect, and the stub in the test was never actually hit. This commit moves the error handling to the right place and updates the specs to ensure the stub is always used.
659 lines
18 KiB
Ruby
659 lines
18 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require "digest/sha1"
|
|
|
|
class Upload < ActiveRecord::Base
|
|
include ActionView::Helpers::NumberHelper
|
|
include HasUrl
|
|
|
|
SHA1_LENGTH = 40
|
|
SEEDED_ID_THRESHOLD = 0
|
|
URL_REGEX ||= /(\/original\/\dX[\/\.\w]*\/(\h+)[\.\w]*)/
|
|
MAX_IDENTIFY_SECONDS = 5
|
|
DOMINANT_COLOR_COMMAND_TIMEOUT_SECONDS = 5
|
|
|
|
belongs_to :user
|
|
belongs_to :access_control_post, class_name: 'Post'
|
|
|
|
# when we access this post we don't care if the post
|
|
# is deleted
|
|
def access_control_post
|
|
Post.unscoped { super }
|
|
end
|
|
|
|
has_many :post_hotlinked_media, dependent: :destroy, class_name: "PostHotlinkedMedia"
|
|
has_many :optimized_images, dependent: :destroy
|
|
has_many :user_uploads, dependent: :destroy
|
|
has_many :upload_references, dependent: :destroy
|
|
has_many :posts, through: :upload_references, source: :target, source_type: 'Post'
|
|
has_many :topic_thumbnails
|
|
|
|
attr_accessor :for_group_message
|
|
attr_accessor :for_theme
|
|
attr_accessor :for_private_message
|
|
attr_accessor :for_export
|
|
attr_accessor :for_site_setting
|
|
attr_accessor :for_gravatar
|
|
|
|
validates_presence_of :filesize
|
|
validates_presence_of :original_filename
|
|
validates :dominant_color, length: { is: 6 }, allow_blank: true, allow_nil: true
|
|
|
|
validates_with UploadValidator
|
|
|
|
before_destroy do
|
|
UserProfile.where(card_background_upload_id: self.id).update_all(card_background_upload_id: nil)
|
|
UserProfile.where(profile_background_upload_id: self.id).update_all(profile_background_upload_id: nil)
|
|
end
|
|
|
|
after_destroy do
|
|
User.where(uploaded_avatar_id: self.id).update_all(uploaded_avatar_id: nil)
|
|
UserAvatar.where(gravatar_upload_id: self.id).update_all(gravatar_upload_id: nil)
|
|
UserAvatar.where(custom_upload_id: self.id).update_all(custom_upload_id: nil)
|
|
end
|
|
|
|
scope :by_users, -> { where("uploads.id > ?", SEEDED_ID_THRESHOLD) }
|
|
|
|
def self.verification_statuses
|
|
@verification_statuses ||= Enum.new(
|
|
unchecked: 1,
|
|
verified: 2,
|
|
invalid_etag: 3
|
|
)
|
|
end
|
|
|
|
def self.add_unused_callback(&block)
|
|
(@unused_callbacks ||= []) << block
|
|
end
|
|
|
|
def self.unused_callbacks
|
|
@unused_callbacks
|
|
end
|
|
|
|
def self.reset_unused_callbacks
|
|
@unused_callbacks = []
|
|
end
|
|
|
|
def self.add_in_use_callback(&block)
|
|
(@in_use_callbacks ||= []) << block
|
|
end
|
|
|
|
def self.in_use_callbacks
|
|
@in_use_callbacks
|
|
end
|
|
|
|
def self.reset_in_use_callbacks
|
|
@in_use_callbacks = []
|
|
end
|
|
|
|
def self.with_no_non_post_relations
|
|
self
|
|
.joins("LEFT JOIN upload_references ur ON ur.upload_id = uploads.id AND ur.target_type != 'Post'")
|
|
.where("ur.upload_id IS NULL")
|
|
end
|
|
|
|
def to_s
|
|
self.url
|
|
end
|
|
|
|
def thumbnail(width = self.thumbnail_width, height = self.thumbnail_height)
|
|
optimized_images.find_by(width: width, height: height)
|
|
end
|
|
|
|
def has_thumbnail?(width, height)
|
|
thumbnail(width, height).present?
|
|
end
|
|
|
|
def create_thumbnail!(width, height, opts = nil)
|
|
return unless SiteSetting.create_thumbnails?
|
|
opts ||= {}
|
|
|
|
if get_optimized_image(width, height, opts)
|
|
save(validate: false)
|
|
end
|
|
end
|
|
|
|
# this method attempts to correct old incorrect extensions
|
|
def get_optimized_image(width, height, opts = nil)
|
|
opts ||= {}
|
|
|
|
if (!extension || extension.length == 0)
|
|
fix_image_extension
|
|
end
|
|
|
|
opts = opts.merge(raise_on_error: true)
|
|
begin
|
|
OptimizedImage.create_for(self, width, height, opts)
|
|
rescue => ex
|
|
Rails.logger.info ex if Rails.env.development?
|
|
opts = opts.merge(raise_on_error: false)
|
|
if fix_image_extension
|
|
OptimizedImage.create_for(self, width, height, opts)
|
|
else
|
|
nil
|
|
end
|
|
end
|
|
end
|
|
|
|
def content
|
|
original_path = Discourse.store.path_for(self)
|
|
external_copy = nil
|
|
|
|
if original_path.blank?
|
|
external_copy = Discourse.store.download(self)
|
|
original_path = external_copy.path
|
|
end
|
|
|
|
File.read(original_path)
|
|
ensure
|
|
if external_copy
|
|
File.unlink(external_copy.path)
|
|
end
|
|
end
|
|
|
|
def fix_image_extension
|
|
return false if extension == "unknown"
|
|
|
|
begin
|
|
# this is relatively cheap once cached
|
|
original_path = Discourse.store.path_for(self)
|
|
if original_path.blank?
|
|
external_copy = Discourse.store.download(self) rescue nil
|
|
original_path = external_copy.try(:path)
|
|
end
|
|
|
|
image_info = FastImage.new(original_path) rescue nil
|
|
new_extension = image_info&.type&.to_s || "unknown"
|
|
|
|
if new_extension != self.extension
|
|
self.update_columns(extension: new_extension)
|
|
true
|
|
end
|
|
rescue
|
|
self.update_columns(extension: "unknown")
|
|
true
|
|
end
|
|
end
|
|
|
|
def destroy
|
|
Upload.transaction do
|
|
Discourse.store.remove_upload(self)
|
|
super
|
|
end
|
|
end
|
|
|
|
def short_url
|
|
"upload://#{short_url_basename}"
|
|
end
|
|
|
|
def uploaded_before_secure_uploads_enabled?
|
|
original_sha1.blank?
|
|
end
|
|
|
|
def matching_access_control_post?(post)
|
|
access_control_post_id == post.id
|
|
end
|
|
|
|
def copied_from_other_post?(post)
|
|
return false if access_control_post_id.blank?
|
|
!matching_access_control_post?(post)
|
|
end
|
|
|
|
def short_path
|
|
self.class.short_path(sha1: self.sha1, extension: self.extension)
|
|
end
|
|
|
|
def self.consider_for_reuse(upload, post)
|
|
return upload if !SiteSetting.secure_uploads? || upload.blank? || post.blank?
|
|
return nil if !upload.matching_access_control_post?(post) || upload.uploaded_before_secure_uploads_enabled?
|
|
upload
|
|
end
|
|
|
|
def self.secure_uploads_url?(url)
|
|
# we do not want to exclude topic links that for whatever reason
|
|
# have secure-uploads in the URL e.g. /t/secure-uploads-are-cool/223452
|
|
route = UrlHelper.rails_route_from_url(url)
|
|
return false if route.blank?
|
|
route[:action] == "show_secure" && route[:controller] == "uploads" && FileHelper.is_supported_media?(url)
|
|
rescue ActionController::RoutingError
|
|
false
|
|
end
|
|
|
|
def self.signed_url_from_secure_uploads_url(url)
|
|
route = UrlHelper.rails_route_from_url(url)
|
|
url = Rails.application.routes.url_for(route.merge(only_path: true))
|
|
secure_upload_s3_path = url[url.index(route[:path])..-1]
|
|
Discourse.store.signed_url_for_path(secure_upload_s3_path)
|
|
end
|
|
|
|
def self.secure_uploads_url_from_upload_url(url)
|
|
return url if !url.include?(SiteSetting.Upload.absolute_base_url)
|
|
uri = URI.parse(url)
|
|
Rails.application.routes.url_for(
|
|
controller: "uploads",
|
|
action: "show_secure",
|
|
path: uri.path[1..-1],
|
|
only_path: true
|
|
)
|
|
end
|
|
|
|
def self.short_path(sha1:, extension:)
|
|
@url_helpers ||= Rails.application.routes.url_helpers
|
|
|
|
@url_helpers.upload_short_path(
|
|
base62: self.base62_sha1(sha1),
|
|
extension: extension
|
|
)
|
|
end
|
|
|
|
def self.base62_sha1(sha1)
|
|
Base62.encode(sha1.hex)
|
|
end
|
|
|
|
def base62_sha1
|
|
Upload.base62_sha1(self.sha1)
|
|
end
|
|
|
|
def local?
|
|
!(url =~ /^(https?:)?\/\//)
|
|
end
|
|
|
|
def fix_dimensions!
|
|
return if !FileHelper.is_supported_image?("image.#{extension}")
|
|
|
|
path =
|
|
if local?
|
|
Discourse.store.path_for(self)
|
|
else
|
|
Discourse.store.download(self).path
|
|
end
|
|
|
|
begin
|
|
if extension == 'svg'
|
|
w, h = Discourse::Utils.execute_command("identify", "-format", "%w %h", path, timeout: MAX_IDENTIFY_SECONDS).split(' ') rescue [0, 0]
|
|
else
|
|
w, h = FastImage.new(path, raise_on_failure: true).size
|
|
end
|
|
|
|
self.width = w || 0
|
|
self.height = h || 0
|
|
|
|
self.thumbnail_width, self.thumbnail_height = ImageSizer.resize(w, h)
|
|
|
|
self.update_columns(
|
|
width: width,
|
|
height: height,
|
|
thumbnail_width: thumbnail_width,
|
|
thumbnail_height: thumbnail_height
|
|
)
|
|
rescue => e
|
|
Discourse.warn_exception(e, message: "Error getting image dimensions")
|
|
end
|
|
nil
|
|
end
|
|
|
|
# on demand image size calculation, this allows us to null out image sizes
|
|
# and still handle as needed
|
|
def get_dimension(key)
|
|
if v = read_attribute(key)
|
|
return v
|
|
end
|
|
fix_dimensions!
|
|
read_attribute(key)
|
|
end
|
|
|
|
def width
|
|
get_dimension(:width)
|
|
end
|
|
|
|
def height
|
|
get_dimension(:height)
|
|
end
|
|
|
|
def thumbnail_width
|
|
get_dimension(:thumbnail_width)
|
|
end
|
|
|
|
def thumbnail_height
|
|
get_dimension(:thumbnail_height)
|
|
end
|
|
|
|
def dominant_color(calculate_if_missing: false)
|
|
val = read_attribute(:dominant_color)
|
|
if val.nil? && calculate_if_missing
|
|
calculate_dominant_color!
|
|
read_attribute(:dominant_color)
|
|
else
|
|
val
|
|
end
|
|
end
|
|
|
|
def calculate_dominant_color!(local_path = nil)
|
|
color = nil
|
|
|
|
if !FileHelper.is_supported_image?("image.#{extension}") || extension == "svg"
|
|
color = ""
|
|
end
|
|
|
|
if color.nil?
|
|
local_path ||=
|
|
if local?
|
|
Discourse.store.path_for(self)
|
|
else
|
|
begin
|
|
Discourse.store.download(self)&.path
|
|
rescue OpenURI::HTTPError => e
|
|
# Some issue with downloading the image from a remote store.
|
|
# Assume the upload is broken and save an empty string to prevent re-evaluation
|
|
nil
|
|
end
|
|
end
|
|
|
|
if local_path.nil?
|
|
# Download failed. Could be too large to download, or file could be missing in s3
|
|
color = ""
|
|
end
|
|
|
|
color ||= begin
|
|
data = Discourse::Utils.execute_command(
|
|
"nice",
|
|
"-n",
|
|
"10",
|
|
"convert",
|
|
local_path,
|
|
"-resize",
|
|
"1x1",
|
|
"-define",
|
|
"histogram:unique-colors=true",
|
|
"-format",
|
|
"%c",
|
|
"histogram:info:",
|
|
timeout: DOMINANT_COLOR_COMMAND_TIMEOUT_SECONDS
|
|
)
|
|
|
|
# Output format:
|
|
# 1: (110.873,116.226,93.8821) #6F745E srgb(43.4798%,45.5789%,36.8165%)
|
|
|
|
color = data[/#([0-9A-F]{6})/, 1]
|
|
|
|
raise "Calculated dominant color but unable to parse output:\n#{data}" if color.nil?
|
|
|
|
color
|
|
rescue Discourse::Utils::CommandError => e
|
|
# Timeout or unable to parse image
|
|
# This can happen due to bad user input - ignore and save
|
|
# an empty string to prevent re-evaluation
|
|
""
|
|
end
|
|
end
|
|
|
|
if persisted?
|
|
self.update_column(:dominant_color, color)
|
|
else
|
|
self.dominant_color = color
|
|
end
|
|
end
|
|
|
|
def target_image_quality(local_path, test_quality)
|
|
@file_quality ||= Discourse::Utils.execute_command("identify", "-format", "%Q", local_path, timeout: MAX_IDENTIFY_SECONDS).to_i rescue 0
|
|
|
|
if @file_quality == 0 || @file_quality > test_quality
|
|
test_quality
|
|
end
|
|
end
|
|
|
|
def self.sha1_from_short_path(path)
|
|
if path =~ /(\/uploads\/short-url\/)([a-zA-Z0-9]+)(\..*)?/
|
|
self.sha1_from_base62_encoded($2)
|
|
end
|
|
end
|
|
|
|
def self.sha1_from_short_url(url)
|
|
if url =~ /(upload:\/\/)?([a-zA-Z0-9]+)(\..*)?/
|
|
self.sha1_from_base62_encoded($2)
|
|
end
|
|
end
|
|
|
|
def self.sha1_from_base62_encoded(encoded_sha1)
|
|
sha1 = Base62.decode(encoded_sha1).to_s(16)
|
|
|
|
if sha1.length > SHA1_LENGTH
|
|
nil
|
|
else
|
|
sha1.rjust(SHA1_LENGTH, '0')
|
|
end
|
|
end
|
|
|
|
def self.generate_digest(path)
|
|
Digest::SHA1.file(path).hexdigest
|
|
end
|
|
|
|
def human_filesize
|
|
number_to_human_size(self.filesize)
|
|
end
|
|
|
|
def rebake_posts_on_old_scheme
|
|
self.posts.where("cooked LIKE '%/_optimized/%'").find_each(&:rebake!)
|
|
end
|
|
|
|
def update_secure_status(source: "unknown", override: nil)
|
|
if override.nil?
|
|
mark_secure, reason = UploadSecurity.new(self).should_be_secure_with_reason
|
|
else
|
|
mark_secure = override
|
|
reason = "manually overridden"
|
|
end
|
|
|
|
secure_status_did_change = self.secure? != mark_secure
|
|
self.update(secure_params(mark_secure, reason, source))
|
|
|
|
if Discourse.store.external?
|
|
begin
|
|
Discourse.store.update_upload_ACL(self)
|
|
rescue Aws::S3::Errors::NotImplemented => err
|
|
Discourse.warn_exception(err, message: "The file store object storage provider does not support setting ACLs")
|
|
end
|
|
end
|
|
|
|
secure_status_did_change
|
|
end
|
|
|
|
def secure_params(secure, reason, source = "unknown")
|
|
{
|
|
secure: secure,
|
|
security_last_changed_reason: reason + " | source: #{source}",
|
|
security_last_changed_at: Time.zone.now
|
|
}
|
|
end
|
|
|
|
def self.migrate_to_new_scheme(limit: nil)
|
|
problems = []
|
|
|
|
DistributedMutex.synchronize("migrate_upload_to_new_scheme") do
|
|
if SiteSetting.migrate_to_new_scheme
|
|
max_file_size_kb = [
|
|
SiteSetting.max_image_size_kb,
|
|
SiteSetting.max_attachment_size_kb
|
|
].max.kilobytes
|
|
|
|
local_store = FileStore::LocalStore.new
|
|
db = RailsMultisite::ConnectionManagement.current_db
|
|
|
|
scope = Upload.by_users
|
|
.where("url NOT LIKE '%/original/_X/%' AND url LIKE '%/uploads/#{db}%'")
|
|
.order(id: :desc)
|
|
|
|
scope = scope.limit(limit) if limit
|
|
|
|
if scope.count == 0
|
|
SiteSetting.migrate_to_new_scheme = false
|
|
return problems
|
|
end
|
|
|
|
remap_scope = nil
|
|
|
|
scope.each do |upload|
|
|
begin
|
|
# keep track of the url
|
|
previous_url = upload.url.dup
|
|
# where is the file currently stored?
|
|
external = previous_url =~ /^\/\//
|
|
# download if external
|
|
if external
|
|
url = SiteSetting.scheme + ":" + previous_url
|
|
|
|
begin
|
|
retries ||= 0
|
|
|
|
file = FileHelper.download(
|
|
url,
|
|
max_file_size: max_file_size_kb,
|
|
tmp_file_name: "discourse",
|
|
follow_redirect: true
|
|
)
|
|
rescue OpenURI::HTTPError
|
|
retry if (retries += 1) < 1
|
|
next
|
|
end
|
|
|
|
path = file.path
|
|
else
|
|
path = local_store.path_for(upload)
|
|
end
|
|
# compute SHA if missing
|
|
if upload.sha1.blank?
|
|
upload.sha1 = Upload.generate_digest(path)
|
|
end
|
|
|
|
# store to new location & update the filesize
|
|
File.open(path) do |f|
|
|
upload.url = Discourse.store.store_upload(f, upload)
|
|
upload.filesize = f.size
|
|
upload.save!(validate: false)
|
|
end
|
|
# remap the URLs
|
|
DbHelper.remap(UrlHelper.absolute(previous_url), upload.url) unless external
|
|
|
|
DbHelper.remap(
|
|
previous_url,
|
|
upload.url,
|
|
excluded_tables: %w{
|
|
posts
|
|
post_search_data
|
|
incoming_emails
|
|
notifications
|
|
single_sign_on_records
|
|
stylesheet_cache
|
|
topic_search_data
|
|
users
|
|
user_emails
|
|
draft_sequences
|
|
optimized_images
|
|
}
|
|
)
|
|
|
|
remap_scope ||= begin
|
|
Post.with_deleted
|
|
.where("raw ~ '/uploads/#{db}/\\d+/' OR raw ~ '/uploads/#{db}/original/(\\d|[a-z])/'")
|
|
.select(:id, :raw, :cooked)
|
|
.all
|
|
end
|
|
|
|
remap_scope.each do |post|
|
|
post.raw.gsub!(previous_url, upload.url)
|
|
post.cooked.gsub!(previous_url, upload.url)
|
|
Post.with_deleted.where(id: post.id).update_all(raw: post.raw, cooked: post.cooked) if post.changed?
|
|
end
|
|
|
|
upload.optimized_images.find_each(&:destroy!)
|
|
upload.rebake_posts_on_old_scheme
|
|
# remove the old file (when local)
|
|
unless external
|
|
FileUtils.rm(path, force: true)
|
|
end
|
|
rescue => e
|
|
problems << { upload: upload, ex: e }
|
|
ensure
|
|
file&.unlink
|
|
file&.close
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
problems
|
|
end
|
|
|
|
def self.extract_upload_ids(raw)
|
|
return [] if raw.blank?
|
|
|
|
sha1s = []
|
|
|
|
raw.scan(/\/(\h{40})/).each do |match|
|
|
sha1s << match[0]
|
|
end
|
|
|
|
raw.scan(/\/([a-zA-Z0-9]+)/).each do |match|
|
|
sha1s << Upload.sha1_from_base62_encoded(match[0])
|
|
end
|
|
|
|
Upload.where(sha1: sha1s.uniq).pluck(:id)
|
|
end
|
|
|
|
def self.backfill_dominant_colors!(count)
|
|
Upload.where(dominant_color: nil).order("id desc").first(count).each do |upload|
|
|
upload.calculate_dominant_color!
|
|
end
|
|
end
|
|
|
|
private
|
|
|
|
def short_url_basename
|
|
"#{Upload.base62_sha1(sha1)}#{extension.present? ? ".#{extension}" : ""}"
|
|
end
|
|
|
|
end
|
|
|
|
# == Schema Information
|
|
#
|
|
# Table name: uploads
|
|
#
|
|
# id :integer not null, primary key
|
|
# user_id :integer not null
|
|
# original_filename :string not null
|
|
# filesize :bigint not null
|
|
# width :integer
|
|
# height :integer
|
|
# url :string not null
|
|
# created_at :datetime not null
|
|
# updated_at :datetime not null
|
|
# sha1 :string(40)
|
|
# origin :string(1000)
|
|
# retain_hours :integer
|
|
# extension :string(10)
|
|
# thumbnail_width :integer
|
|
# thumbnail_height :integer
|
|
# etag :string
|
|
# secure :boolean default(FALSE), not null
|
|
# access_control_post_id :bigint
|
|
# original_sha1 :string
|
|
# animated :boolean
|
|
# verification_status :integer default(1), not null
|
|
# security_last_changed_at :datetime
|
|
# security_last_changed_reason :string
|
|
# dominant_color :text
|
|
#
|
|
# Indexes
|
|
#
|
|
# idx_uploads_on_verification_status (verification_status)
|
|
# index_uploads_on_access_control_post_id (access_control_post_id)
|
|
# index_uploads_on_etag (etag)
|
|
# index_uploads_on_extension (lower((extension)::text))
|
|
# index_uploads_on_id (id) WHERE (dominant_color IS NULL)
|
|
# index_uploads_on_id_and_url (id,url)
|
|
# index_uploads_on_original_sha1 (original_sha1)
|
|
# index_uploads_on_sha1 (sha1) UNIQUE
|
|
# index_uploads_on_url (url)
|
|
# index_uploads_on_user_id (user_id)
|
|
#
|