mirror of
https://github.com/discourse/discourse.git
synced 2024-11-22 11:44:49 +08:00
c69b58e22e
The other processing operations, such as fixing orientation or cropping, can in rare cases increase the size of the uploaded image. Running the downsize step after all these operations should create the best image possible.
484 lines
16 KiB
Ruby
484 lines
16 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require "fastimage"
|
|
|
|
class UploadCreator
|
|
|
|
TYPES_TO_CROP ||= %w{avatar card_background custom_emoji profile_background}.each(&:freeze)
|
|
|
|
ALLOWED_SVG_ELEMENTS ||= %w{
|
|
circle clipPath defs ellipse feGaussianBlur filter g line linearGradient
|
|
marker path polygon polyline radialGradient rect stop style svg text
|
|
textPath tref tspan use
|
|
}.each(&:freeze)
|
|
|
|
include ActiveSupport::Deprecation::DeprecatedConstantAccessor
|
|
deprecate_constant 'WHITELISTED_SVG_ELEMENTS', 'UploadCreator::ALLOWED_SVG_ELEMENTS'
|
|
|
|
# Available options
|
|
# - type (string)
|
|
# - origin (string)
|
|
# - for_group_message (boolean)
|
|
# - for_theme (boolean)
|
|
# - for_private_message (boolean)
|
|
# - pasted (boolean)
|
|
# - for_export (boolean)
|
|
# - for_gravatar (boolean)
|
|
# - skip_validations (boolean)
|
|
def initialize(file, filename, opts = {})
|
|
@file = file
|
|
@filename = (filename || "").gsub(/[^[:print:]]/, "")
|
|
@upload = Upload.new(original_filename: @filename, filesize: 0)
|
|
@opts = opts
|
|
@opts[:validate] = opts[:skip_validations].present? ? !ActiveRecord::Type::Boolean.new.cast(opts[:skip_validations]) : true
|
|
end
|
|
|
|
def create_for(user_id)
|
|
if filesize <= 0
|
|
@upload.errors.add(:base, I18n.t("upload.empty"))
|
|
return @upload
|
|
end
|
|
|
|
@image_info = FastImage.new(@file) rescue nil
|
|
is_image = FileHelper.is_supported_image?(@filename)
|
|
is_image ||= @image_info && FileHelper.is_supported_image?("test.#{@image_info.type}")
|
|
is_image = false if @opts[:for_theme]
|
|
|
|
DistributedMutex.synchronize("upload_#{user_id}_#{@filename}") do
|
|
# We need to convert HEIFs early because FastImage does not consider them as images
|
|
if convert_heif_to_jpeg?
|
|
convert_heif!
|
|
is_image = FileHelper.is_supported_image?("test.#{@image_info.type}")
|
|
end
|
|
|
|
if is_image
|
|
extract_image_info!
|
|
return @upload if @upload.errors.present?
|
|
|
|
if @image_info.type.to_s == "svg"
|
|
clean_svg!
|
|
elsif !Rails.env.test? || @opts[:force_optimize]
|
|
convert_to_jpeg! if convert_png_to_jpeg? || should_alter_quality?
|
|
fix_orientation! if should_fix_orientation?
|
|
crop! if should_crop?
|
|
optimize! if should_optimize?
|
|
downsize! if should_downsize?
|
|
return @upload if is_still_too_big?
|
|
end
|
|
|
|
# conversion may have switched the type
|
|
image_type = @image_info.type.to_s
|
|
end
|
|
|
|
# compute the sha of the file and generate a unique hash
|
|
# which is only used for secure uploads
|
|
sha1 = Upload.generate_digest(@file)
|
|
unique_hash = SecureRandom.hex(20) if SiteSetting.secure_media
|
|
|
|
# we do not check for duplicate uploads if secure media is
|
|
# enabled because we use a unique access hash to differentiate
|
|
# between uploads instead of the sha1, and to get around various
|
|
# access/permission issues for uploads
|
|
if !SiteSetting.secure_media
|
|
# do we already have that upload?
|
|
@upload = Upload.find_by(sha1: sha1)
|
|
|
|
# make sure the previous upload has not failed
|
|
if @upload && @upload.url.blank?
|
|
@upload.destroy
|
|
@upload = nil
|
|
end
|
|
|
|
# return the previous upload if any
|
|
if @upload
|
|
add_metadata!
|
|
UserUpload.find_or_create_by!(user_id: user_id, upload_id: @upload.id) if user_id
|
|
return @upload
|
|
end
|
|
end
|
|
|
|
fixed_original_filename = nil
|
|
|
|
if is_image
|
|
current_extension = File.extname(@filename).downcase.sub("jpeg", "jpg")
|
|
expected_extension = ".#{image_type}".downcase.sub("jpeg", "jpg")
|
|
|
|
# we have to correct original filename here, no choice
|
|
# otherwise validation will fail and we can not save
|
|
# TODO decide if we only run the validation on the extension
|
|
if current_extension != expected_extension
|
|
basename = File.basename(@filename, current_extension).presence || "image"
|
|
fixed_original_filename = "#{basename}#{expected_extension}"
|
|
end
|
|
end
|
|
|
|
# create the upload otherwise
|
|
@upload = Upload.new
|
|
@upload.user_id = user_id
|
|
@upload.original_filename = fixed_original_filename || @filename
|
|
@upload.filesize = filesize
|
|
@upload.sha1 = SiteSetting.secure_media? ? unique_hash : sha1
|
|
@upload.original_sha1 = SiteSetting.secure_media? ? sha1 : nil
|
|
@upload.url = ""
|
|
@upload.origin = @opts[:origin][0...1000] if @opts[:origin]
|
|
@upload.extension = image_type || File.extname(@filename)[1..10]
|
|
|
|
if is_image
|
|
if @image_info.type.to_s == 'svg'
|
|
w, h = [0, 0]
|
|
|
|
begin
|
|
w, h = Discourse::Utils
|
|
.execute_command("identify", "-ping", "-format", "%w %h", @file.path, timeout: Upload::MAX_IDENTIFY_SECONDS)
|
|
.split(' ')
|
|
rescue
|
|
# use default 0, 0
|
|
end
|
|
else
|
|
w, h = @image_info.size
|
|
end
|
|
|
|
@upload.thumbnail_width, @upload.thumbnail_height = ImageSizer.resize(w, h)
|
|
@upload.width, @upload.height = w, h
|
|
@upload.animated = animated?
|
|
end
|
|
|
|
add_metadata!
|
|
|
|
if SiteSetting.secure_media
|
|
secure, reason = UploadSecurity.new(@upload, @opts.merge(creating: true)).should_be_secure_with_reason
|
|
attrs = @upload.secure_params(secure, reason, "upload creator")
|
|
@upload.assign_attributes(attrs)
|
|
end
|
|
|
|
# Callbacks using this event to validate the upload or the file must add errors to the
|
|
# upload errors object.
|
|
DiscourseEvent.trigger(:before_upload_creation, @file, is_image, @upload, @opts[:validate])
|
|
return @upload unless @upload.errors.empty? && @upload.save(validate: @opts[:validate])
|
|
|
|
# store the file and update its url
|
|
File.open(@file.path) do |f|
|
|
url = Discourse.store.store_upload(f, @upload)
|
|
|
|
if url.present?
|
|
@upload.url = url
|
|
@upload.save!(validate: @opts[:validate])
|
|
else
|
|
@upload.errors.add(:url, I18n.t("upload.store_failure", upload_id: @upload.id, user_id: user_id))
|
|
end
|
|
end
|
|
|
|
if @upload.errors.empty? && is_image && @opts[:type] == "avatar" && @upload.extension != "svg"
|
|
Jobs.enqueue(:create_avatar_thumbnails, upload_id: @upload.id)
|
|
end
|
|
|
|
if @upload.errors.empty?
|
|
UserUpload.find_or_create_by!(user_id: user_id, upload_id: @upload.id) if user_id
|
|
end
|
|
|
|
@upload
|
|
end
|
|
ensure
|
|
if @file
|
|
@file.respond_to?(:close!) ? @file.close! : @file.close
|
|
end
|
|
end
|
|
|
|
def extract_image_info!
|
|
@image_info = FastImage.new(@file) rescue nil
|
|
@file.rewind
|
|
|
|
if @image_info.nil?
|
|
@upload.errors.add(:base, I18n.t("upload.images.not_supported_or_corrupted"))
|
|
elsif filesize <= 0
|
|
@upload.errors.add(:base, I18n.t("upload.empty"))
|
|
elsif pixels == 0 && @image_info.type.to_s != 'svg'
|
|
@upload.errors.add(:base, I18n.t("upload.images.size_not_found"))
|
|
elsif max_image_pixels > 0 && pixels >= max_image_pixels * 2
|
|
@upload.errors.add(:base, I18n.t("upload.images.larger_than_x_megapixels", max_image_megapixels: SiteSetting.max_image_megapixels * 2))
|
|
end
|
|
end
|
|
|
|
MIN_PIXELS_TO_CONVERT_TO_JPEG ||= 1280 * 720
|
|
|
|
def convert_png_to_jpeg?
|
|
return false unless @image_info.type == :png
|
|
return true if @opts[:pasted]
|
|
return false if SiteSetting.png_to_jpg_quality == 100
|
|
pixels > MIN_PIXELS_TO_CONVERT_TO_JPEG
|
|
end
|
|
|
|
MIN_CONVERT_TO_JPEG_BYTES_SAVED = 75_000
|
|
MIN_CONVERT_TO_JPEG_SAVING_RATIO = 0.70
|
|
|
|
def convert_to_jpeg!
|
|
return if @opts[:for_site_setting]
|
|
return if filesize < MIN_CONVERT_TO_JPEG_BYTES_SAVED
|
|
|
|
jpeg_tempfile = Tempfile.new(["image", ".jpg"])
|
|
|
|
from = @file.path
|
|
to = jpeg_tempfile.path
|
|
|
|
OptimizedImage.ensure_safe_paths!(from, to)
|
|
|
|
from = OptimizedImage.prepend_decoder!(from, nil, filename: "image.#{@image_info.type}")
|
|
to = OptimizedImage.prepend_decoder!(to)
|
|
|
|
opts = {}
|
|
desired_quality = [SiteSetting.png_to_jpg_quality, SiteSetting.recompress_original_jpg_quality].compact.min
|
|
target_quality = @upload.target_image_quality(from, desired_quality)
|
|
opts = { quality: target_quality } if target_quality
|
|
|
|
begin
|
|
execute_convert(from, to, opts)
|
|
rescue
|
|
# retry with debugging enabled
|
|
execute_convert(from, to, opts.merge(debug: true))
|
|
end
|
|
|
|
new_size = File.size(jpeg_tempfile.path)
|
|
|
|
keep_jpeg = new_size < filesize * MIN_CONVERT_TO_JPEG_SAVING_RATIO
|
|
keep_jpeg &&= (filesize - new_size) > MIN_CONVERT_TO_JPEG_BYTES_SAVED
|
|
|
|
if keep_jpeg
|
|
@file.respond_to?(:close!) ? @file.close! : @file.close
|
|
@file = jpeg_tempfile
|
|
extract_image_info!
|
|
else
|
|
jpeg_tempfile.close!
|
|
end
|
|
end
|
|
|
|
def convert_heif_to_jpeg?
|
|
File.extname(@filename).downcase.match?(/\.hei(f|c)$/)
|
|
end
|
|
|
|
def convert_heif!
|
|
jpeg_tempfile = Tempfile.new(["image", ".jpg"])
|
|
from = @file.path
|
|
to = jpeg_tempfile.path
|
|
OptimizedImage.ensure_safe_paths!(from, to)
|
|
|
|
begin
|
|
execute_convert(from, to)
|
|
rescue
|
|
# retry with debugging enabled
|
|
execute_convert(from, to, { debug: true })
|
|
end
|
|
|
|
@file.respond_to?(:close!) ? @file.close! : @file.close
|
|
@file = jpeg_tempfile
|
|
extract_image_info!
|
|
end
|
|
|
|
MAX_CONVERT_FORMAT_SECONDS = 20
|
|
def execute_convert(from, to, opts = {})
|
|
command = [
|
|
"convert",
|
|
from,
|
|
"-auto-orient",
|
|
"-background", "white",
|
|
"-interlace", "none",
|
|
"-flatten"
|
|
]
|
|
command << "-debug" << "all" if opts[:debug]
|
|
command << "-quality" << opts[:quality].to_s if opts[:quality]
|
|
command << to
|
|
|
|
Discourse::Utils.execute_command(
|
|
*command,
|
|
failure_message: I18n.t("upload.png_to_jpg_conversion_failure_message"),
|
|
timeout: MAX_CONVERT_FORMAT_SECONDS
|
|
)
|
|
end
|
|
|
|
def should_alter_quality?
|
|
return false if animated?
|
|
|
|
desired_quality = @image_info.type == :png ? SiteSetting.png_to_jpg_quality : SiteSetting.recompress_original_jpg_quality
|
|
@upload.target_image_quality(@file.path, desired_quality).present?
|
|
end
|
|
|
|
def should_downsize?
|
|
max_image_size > 0 && filesize >= max_image_size && !animated?
|
|
end
|
|
|
|
def downsize!
|
|
3.times do
|
|
original_size = filesize
|
|
down_tempfile = Tempfile.new(["down", ".#{@image_info.type}"])
|
|
|
|
from = @file.path
|
|
to = down_tempfile.path
|
|
|
|
OptimizedImage.ensure_safe_paths!(from, to)
|
|
|
|
OptimizedImage.downsize(
|
|
from,
|
|
to,
|
|
"50%",
|
|
scale_image: true,
|
|
raise_on_error: true
|
|
)
|
|
|
|
@file.respond_to?(:close!) ? @file.close! : @file.close
|
|
@file = down_tempfile
|
|
|
|
extract_image_info!
|
|
|
|
return if filesize >= original_size || pixels == 0 || !should_downsize?
|
|
end
|
|
rescue
|
|
@upload.errors.add(:base, I18n.t("upload.optimize_failure_message"))
|
|
end
|
|
|
|
def is_still_too_big?
|
|
if max_image_pixels > 0 && pixels >= max_image_pixels
|
|
@upload.errors.add(:base, I18n.t("upload.images.larger_than_x_megapixels", max_image_megapixels: SiteSetting.max_image_megapixels))
|
|
true
|
|
elsif max_image_size > 0 && filesize >= max_image_size
|
|
@upload.errors.add(:base, I18n.t("upload.images.too_large", max_size_kb: SiteSetting.max_image_size_kb))
|
|
true
|
|
else
|
|
false
|
|
end
|
|
end
|
|
|
|
def clean_svg!
|
|
doc = Nokogiri::XML(@file)
|
|
doc.xpath(svg_allowlist_xpath).remove
|
|
doc.xpath("//@*[starts-with(name(), 'on')]").remove
|
|
doc.css('use').each do |use_el|
|
|
if use_el.attr('href')
|
|
use_el.remove_attribute('href') unless use_el.attr('href').starts_with?('#')
|
|
end
|
|
if use_el.attr('xlink:href')
|
|
use_el.remove_attribute('xlink:href') unless use_el.attr('xlink:href').starts_with?('#')
|
|
end
|
|
end
|
|
File.write(@file.path, doc.to_s)
|
|
@file.rewind
|
|
end
|
|
|
|
def should_fix_orientation?
|
|
# orientation is between 1 and 8, 1 being the default
|
|
# cf. http://www.daveperrett.com/articles/2012/07/28/exif-orientation-handling-is-a-ghetto/
|
|
@image_info.orientation.to_i > 1
|
|
end
|
|
|
|
MAX_FIX_ORIENTATION_TIME = 5
|
|
def fix_orientation!
|
|
path = @file.path
|
|
|
|
OptimizedImage.ensure_safe_paths!(path)
|
|
path = OptimizedImage.prepend_decoder!(path, nil, filename: "image.#{@image_info.type}")
|
|
|
|
Discourse::Utils.execute_command('convert', path, '-auto-orient', path, timeout: MAX_FIX_ORIENTATION_TIME)
|
|
|
|
extract_image_info!
|
|
end
|
|
|
|
def should_crop?
|
|
return false if ['profile_background', 'card_background', 'custom_emoji'].include?(@opts[:type]) && animated?
|
|
|
|
TYPES_TO_CROP.include?(@opts[:type])
|
|
end
|
|
|
|
def crop!
|
|
max_pixel_ratio = Discourse::PIXEL_RATIOS.max
|
|
filename_with_correct_ext = "image.#{@image_info.type}"
|
|
|
|
case @opts[:type]
|
|
when "avatar"
|
|
width = height = Discourse.avatar_sizes.max
|
|
OptimizedImage.resize(@file.path, @file.path, width, height, filename: filename_with_correct_ext)
|
|
when "profile_background"
|
|
max_width = 850 * max_pixel_ratio
|
|
width, height = ImageSizer.resize(@image_info.size[0], @image_info.size[1], max_width: max_width, max_height: max_width)
|
|
OptimizedImage.downsize(@file.path, @file.path, "#{width}x#{height}\>", filename: filename_with_correct_ext)
|
|
when "card_background"
|
|
max_width = 590 * max_pixel_ratio
|
|
width, height = ImageSizer.resize(@image_info.size[0], @image_info.size[1], max_width: max_width, max_height: max_width)
|
|
OptimizedImage.downsize(@file.path, @file.path, "#{width}x#{height}\>", filename: filename_with_correct_ext)
|
|
when "custom_emoji"
|
|
OptimizedImage.downsize(@file.path, @file.path, "100x100\>", filename: filename_with_correct_ext)
|
|
end
|
|
|
|
extract_image_info!
|
|
end
|
|
|
|
def should_optimize?
|
|
# GIF is too slow (plus, we'll soon be converting them to MP4)
|
|
# Optimizing SVG is useless
|
|
return false if @file.path =~ /\.(gif|svg)$/i
|
|
# Safeguard for large PNGs
|
|
return pixels < 2_000_000 if @file.path =~ /\.png/i
|
|
# Everything else is fine!
|
|
true
|
|
end
|
|
|
|
def optimize!
|
|
OptimizedImage.ensure_safe_paths!(@file.path)
|
|
FileHelper.optimize_image!(@file.path)
|
|
extract_image_info!
|
|
end
|
|
|
|
def filesize
|
|
File.size?(@file.path).to_i
|
|
end
|
|
|
|
def max_image_size
|
|
@max_image_size ||= SiteSetting.max_image_size_kb.kilobytes
|
|
end
|
|
|
|
def max_image_pixels
|
|
@max_image_pixels ||= SiteSetting.max_image_megapixels * 1_000_000
|
|
end
|
|
|
|
def pixels
|
|
@image_info.size&.reduce(:*).to_i
|
|
end
|
|
|
|
def svg_allowlist_xpath
|
|
@@svg_allowlist_xpath ||= "//*[#{ALLOWED_SVG_ELEMENTS.map { |e| "name()!='#{e}'" }.join(" and ") }]"
|
|
end
|
|
|
|
def add_metadata!
|
|
@upload.for_private_message = true if @opts[:for_private_message]
|
|
@upload.for_group_message = true if @opts[:for_group_message]
|
|
@upload.for_theme = true if @opts[:for_theme]
|
|
@upload.for_export = true if @opts[:for_export]
|
|
@upload.for_site_setting = true if @opts[:for_site_setting]
|
|
@upload.for_gravatar = true if @opts[:for_gravatar]
|
|
end
|
|
|
|
private
|
|
|
|
def animated?
|
|
return @animated if @animated != nil
|
|
|
|
@animated ||= begin
|
|
is_animated = FastImage.animated?(@file)
|
|
type = @image_info.type.to_s
|
|
|
|
if is_animated != nil
|
|
# FastImage will return nil if it cannot determine if animated
|
|
is_animated
|
|
elsif type == "gif" || type == "webp"
|
|
# Only GIFs, WEBPs and a few other unsupported image types can be animated
|
|
OptimizedImage.ensure_safe_paths!(@file.path)
|
|
|
|
command = ["identify", "-format", "%n\\n", @file.path]
|
|
frames = Discourse::Utils.execute_command(*command, timeout: Upload::MAX_IDENTIFY_SECONDS).to_i rescue 1
|
|
|
|
frames > 1
|
|
else
|
|
false
|
|
end
|
|
end
|
|
end
|
|
|
|
end
|