discourse/app/models/upload.rb

# frozen_string_literal: true

require "digest/sha1"

class Upload < ActiveRecord::Base
  include ActionView::Helpers::NumberHelper
  include HasUrl

  SHA1_LENGTH = 40
  SEEDED_ID_THRESHOLD = 0
  URL_REGEX = %r{(/original/\dX[/\.\w]*/(\h+)[\.\w]*)}
  MAX_IDENTIFY_SECONDS = 5
  DOMINANT_COLOR_COMMAND_TIMEOUT_SECONDS = 5

  belongs_to :user
  belongs_to :access_control_post, class_name: "Post"

  # when we access this post we don't care if the post
  # is deleted
  def access_control_post
    Post.unscoped { super }
  end

  has_many :post_hotlinked_media, dependent: :destroy, class_name: "PostHotlinkedMedia"
  has_many :optimized_images, dependent: :destroy
  has_many :user_uploads, dependent: :destroy
  has_many :upload_references, dependent: :destroy
  has_many :posts, through: :upload_references, source: :target, source_type: "Post"
  has_many :topic_thumbnails
  has_many :badges, foreign_key: :image_upload_id, dependent: :nullify

  attr_accessor :for_group_message
  attr_accessor :for_theme
  attr_accessor :for_private_message
  attr_accessor :for_export
  attr_accessor :for_site_setting
  attr_accessor :for_gravatar
  attr_accessor :validate_file_size

  validates_presence_of :filesize
  validates_presence_of :original_filename
  validates :dominant_color, length: { is: 6 }, allow_blank: true, allow_nil: true

  validates_with UploadValidator

  before_destroy do
    UserProfile.where(card_background_upload_id: self.id).update_all(card_background_upload_id: nil)
    UserProfile.where(profile_background_upload_id: self.id).update_all(
      profile_background_upload_id: nil,
    )
  end

  after_destroy do
    User.where(uploaded_avatar_id: self.id).update_all(uploaded_avatar_id: nil)
    UserAvatar.where(gravatar_upload_id: self.id).update_all(gravatar_upload_id: nil)
    UserAvatar.where(custom_upload_id: self.id).update_all(custom_upload_id: nil)
  end

  scope :by_users, -> { where("uploads.id > ?", SEEDED_ID_THRESHOLD) }

  scope :without_s3_file_missing_confirmed_verification_status,
        -> do
          where.not(verification_status: Upload.verification_statuses[:s3_file_missing_confirmed])
        end

  scope :with_invalid_etag_verification_status,
        -> { where(verification_status: Upload.verification_statuses[:invalid_etag]) }

  def self.verification_statuses
    @verification_statuses ||=
      Enum.new(
        unchecked: 1,
        verified: 2,
        invalid_etag: 3, # Used by S3Inventory to mark S3 Upload records that have an invalid ETag value compared to the ETag value of the inventory file
        s3_file_missing_confirmed: 4, # Used by S3Inventory to skip S3 Upload records that are confirmed to not be backed by a file in the S3 file store
      )
  end

  def self.mark_invalid_s3_uploads_as_missing
    Upload.with_invalid_etag_verification_status.update_all(
      verification_status: Upload.verification_statuses[:s3_file_missing_confirmed],
    )
  end

  def self.add_unused_callback(&block)
    (@unused_callbacks ||= []) << block
  end

  def self.unused_callbacks
    @unused_callbacks
  end

  def self.reset_unused_callbacks
    @unused_callbacks = []
  end

  def self.add_in_use_callback(&block)
    (@in_use_callbacks ||= []) << block
  end

  def self.in_use_callbacks
    @in_use_callbacks
  end

  def self.reset_in_use_callbacks
    @in_use_callbacks = []
  end

  def self.with_no_non_post_relations
    self.joins(
      "LEFT JOIN upload_references ur ON ur.upload_id = uploads.id AND ur.target_type != 'Post'",
    ).where("ur.upload_id IS NULL")
  end

  def initialize(*args)
    super
    self.validate_file_size = true
  end

  def to_s
    self.url
  end

  def to_markdown
    UploadMarkdown.new(self).to_markdown
  end

  def thumbnail(width = self.thumbnail_width, height = self.thumbnail_height)
    optimized_images.find_by(width: width, height: height)
  end

  def has_thumbnail?(width, height)
    thumbnail(width, height).present?
  end

  def create_thumbnail!(width, height, opts = nil)
    return unless SiteSetting.create_thumbnails?
    opts ||= {}

    save(validate: false) if get_optimized_image(width, height, opts)
  end

  # this method attempts to correct old incorrect extensions
  def get_optimized_image(width, height, opts = nil)
    opts ||= {}

    fix_image_extension if (!extension || extension.length == 0)

    opts = opts.merge(raise_on_error: true)
    begin
      OptimizedImage.create_for(self, width, height, opts)
    rescue => ex
      Rails.logger.info ex if Rails.env.development?
      opts = opts.merge(raise_on_error: false)
      if fix_image_extension
        OptimizedImage.create_for(self, width, height, opts)
      else
        nil
      end
    end
  end

  def content
    original_path = Discourse.store.path_for(self)
    external_copy = nil

    if original_path.blank?
      external_copy = Discourse.store.download!(self)
      original_path = external_copy.path
    end

    File.read(original_path)
  ensure
    File.unlink(external_copy.path) if external_copy
  end

  def fix_image_extension
    return false if extension == "unknown"

    begin
      # this is relatively cheap once cached
      original_path = Discourse.store.path_for(self)
      if original_path.blank?
        external_copy = Discourse.store.download_safe(self)
        original_path = external_copy&.path
      end

      image_info =
        begin
          FastImage.new(original_path)
        rescue StandardError
          nil
        end
      new_extension = image_info&.type&.to_s || "unknown"

      if new_extension != self.extension
        self.update_columns(extension: new_extension)
        true
      end
    rescue StandardError
      self.update_columns(extension: "unknown")
      true
    end
  end

  def destroy
    Upload.transaction do
      Discourse.store.remove_upload(self)
      super
    end
  end

  def short_url
    "upload://#{short_url_basename}"
  end

  def uploaded_before_secure_uploads_enabled?
    original_sha1.blank?
  end

  def matching_access_control_post?(post)
    access_control_post_id == post.id
  end

  def copied_from_other_post?(post)
    return false if access_control_post_id.blank?
    !matching_access_control_post?(post)
  end

  def short_path
    self.class.short_path(sha1: self.sha1, extension: self.extension)
  end

  def self.consider_for_reuse(upload, post)
    return upload if !SiteSetting.secure_uploads? || upload.blank? || post.blank?
    if !upload.matching_access_control_post?(post) || upload.uploaded_before_secure_uploads_enabled?
      return nil
    end
    upload
  end

  def self.secure_uploads_url?(url)
    # we do not want to exclude topic links that for whatever reason
    # have secure-uploads in the URL e.g. /t/secure-uploads-are-cool/223452
    route = UrlHelper.rails_route_from_url(url)
    return false if route.blank?
    route[:action] == "show_secure" && route[:controller] == "uploads" &&
      FileHelper.is_supported_media?(url)
  rescue ActionController::RoutingError
    false
  end

  def self.signed_url_from_secure_uploads_url(url)
    route = UrlHelper.rails_route_from_url(url)
    url = Rails.application.routes.url_for(route.merge(only_path: true))
    secure_upload_s3_path = url[url.index(route[:path])..-1]
    Discourse.store.signed_url_for_path(secure_upload_s3_path)
  end

  def self.secure_uploads_url_from_upload_url(url)
    return url if !url.include?(SiteSetting.Upload.absolute_base_url)
    uri = URI.parse(url)
    Rails.application.routes.url_for(
      controller: "uploads",
      action: "show_secure",
      path: uri.path[1..-1],
      only_path: true,
    )
  end

  def self.short_path(sha1:, extension:)
    @url_helpers ||= Rails.application.routes.url_helpers

    @url_helpers.upload_short_path(base62: self.base62_sha1(sha1), extension: extension)
  end

  def self.base62_sha1(sha1)
    Base62.encode(sha1.hex)
  end

  def base62_sha1
    Upload.base62_sha1(self.sha1)
  end

  def local?
    !(url =~ %r{\A(https?:)?//})
  end

  def fix_dimensions!
    return if !FileHelper.is_supported_image?("image.#{extension}")

    begin
      path =
        if local?
          Discourse.store.path_for(self)
        else
          Discourse.store.download!(self).path
        end

      if extension == "svg"
        w, h =
          begin
            Discourse::Utils.execute_command(
              "identify",
              "-ping",
              "-format",
              "%w %h",
              path,
              timeout: MAX_IDENTIFY_SECONDS,
            ).split(" ")
          rescue StandardError
            [0, 0]
          end
      else
        w, h = FastImage.new(path, raise_on_failure: true).size
      end

      self.width = w || 0
      self.height = h || 0

      self.thumbnail_width, self.thumbnail_height = ImageSizer.resize(w, h)

      self.update_columns(
        width: width,
        height: height,
        thumbnail_width: thumbnail_width,
        thumbnail_height: thumbnail_height,
      )
    rescue => e
      Discourse.warn_exception(e, message: "Error getting image dimensions")
    end
    nil
  end

  # on demand image size calculation, this allows us to null out image sizes
  # and still handle as needed
  def get_dimension(key)
    if v = read_attribute(key)
      return v
    end
    fix_dimensions!
    read_attribute(key)
  end

  def width
    get_dimension(:width)
  end

  def height
    get_dimension(:height)
  end

  def thumbnail_width
    get_dimension(:thumbnail_width)
  end

  def thumbnail_height
    get_dimension(:thumbnail_height)
  end

  def dominant_color(calculate_if_missing: false)
    val = read_attribute(:dominant_color)
    if val.nil? && calculate_if_missing
      calculate_dominant_color!
      read_attribute(:dominant_color)
    else
      val
    end
  end

  def calculate_dominant_color!(local_path = nil)
    color = nil

    color = "" if !FileHelper.is_supported_image?("image.#{extension}") || extension == "svg"

    if color.nil?
      local_path ||=
        if local?
          Discourse.store.path_for(self)
        else
          Discourse.store.download_safe(self)&.path
        end

      if local_path.nil?
        # Download failed. Could be too large to download, or file could be missing in s3
        color = ""
      end

      color ||=
        begin
          data =
            Discourse::Utils.execute_command(
              "nice",
              "-n",
              "10",
              "convert",
              local_path,
              "-depth",
              "8",
              "-resize",
              "1x1",
              "-define",
              "histogram:unique-colors=true",
              "-format",
              "%c",
              "histogram:info:",
              timeout: DOMINANT_COLOR_COMMAND_TIMEOUT_SECONDS,
            )

          # Output format:
          # 1: (110.873,116.226,93.8821) #6F745E srgb(43.4798%,45.5789%,36.8165%)

          color = data[/#([0-9A-F]{6})/, 1]

          raise "Calculated dominant color but unable to parse output:\n#{data}" if color.nil?

          color
        rescue Discourse::Utils::CommandError => e
          # Timeout or unable to parse image
          # This can happen due to bad user input - ignore and save
          # an empty string to prevent re-evaluation
          ""
        end
    end

    if persisted?
      self.update_column(:dominant_color, color)
    else
      self.dominant_color = color
    end
  end

  def target_image_quality(local_path, test_quality)
    @file_quality ||=
      begin
        Discourse::Utils.execute_command(
          "identify",
          "-ping",
          "-format",
          "%Q",
          local_path,
          timeout: MAX_IDENTIFY_SECONDS,
        ).to_i
      rescue StandardError
        0
      end

    test_quality if @file_quality == 0 || @file_quality > test_quality
  end

  def self.sha1_from_short_path(path)
    self.sha1_from_base62_encoded($2) if path =~ %r{(/uploads/short-url/)([a-zA-Z0-9]+)(\..*)?}
  end

  def self.sha1_from_short_url(url)
    self.sha1_from_base62_encoded($2) if url =~ %r{(upload://)?([a-zA-Z0-9]+)(\..*)?}
  end

  def self.sha1_from_long_url(url)
    $2 if url =~ URL_REGEX || url =~ OptimizedImage::URL_REGEX
  end

  def self.sha1_from_base62_encoded(encoded_sha1)
    sha1 = Base62.decode(encoded_sha1).to_s(16)

    if sha1.length > SHA1_LENGTH
      nil
    else
      sha1.rjust(SHA1_LENGTH, "0")
    end
  end

  def self.generate_digest(path)
    Digest::SHA1.file(path).hexdigest
  end

  def human_filesize
    number_to_human_size(self.filesize)
  end

  def rebake_posts_on_old_scheme
    self.posts.where("cooked LIKE '%/_optimized/%'").find_each(&:rebake!)
  end

  def update_secure_status(source: "unknown", override: nil)
    if override.nil?
      mark_secure, reason = UploadSecurity.new(self).should_be_secure_with_reason
    else
      mark_secure = override
      reason = "manually overridden"
    end

    secure_status_did_change = self.secure? != mark_secure
    self.update(secure_params(mark_secure, reason, source))

    if secure_status_did_change && SiteSetting.s3_use_acls && Discourse.store.external?
      begin
        Discourse.store.update_upload_ACL(self)
      rescue Aws::S3::Errors::NotImplemented => err
        Discourse.warn_exception(
          err,
          message: "The file store object storage provider does not support setting ACLs",
        )
      end
    end

    secure_status_did_change
  end

  def secure_params(secure, reason, source = "unknown")
    {
      secure: secure,
      security_last_changed_reason: reason + " | source: #{source}",
      security_last_changed_at: Time.zone.now,
    }
  end

  def self.migrate_to_new_scheme(limit: nil)
    problems = []

    DistributedMutex.synchronize("migrate_upload_to_new_scheme") do
      if SiteSetting.migrate_to_new_scheme
        max_file_size_kb = [
          SiteSetting.max_image_size_kb,
          SiteSetting.max_attachment_size_kb,
        ].max.kilobytes

        local_store = FileStore::LocalStore.new
        db = RailsMultisite::ConnectionManagement.current_db

        scope =
          Upload
            .by_users
            .where("url NOT LIKE '%/original/_X/%' AND url LIKE ?", "%/uploads/#{db}%")
            .order(id: :desc)

        scope = scope.limit(limit) if limit

        if scope.count == 0
          SiteSetting.migrate_to_new_scheme = false
          return problems
        end

        remap_scope = nil

        scope.each do |upload|
          begin
            # keep track of the url
            previous_url = upload.url.dup
            # where is the file currently stored?
            external = previous_url =~ %r{\A//}
            # download if external
            if external
              url = SiteSetting.scheme + ":" + previous_url

              begin
                retries ||= 0

                file =
                  FileHelper.download(
                    url,
                    max_file_size: max_file_size_kb,
                    tmp_file_name: "discourse",
                    follow_redirect: true,
                  )
              rescue OpenURI::HTTPError
                retry if (retries += 1) < 1
                next
              end

              path = file.path
            else
              path = local_store.path_for(upload)
            end
            # compute SHA if missing
            upload.sha1 = Upload.generate_digest(path) if upload.sha1.blank?

            # store to new location & update the filesize
            File.open(path) do |f|
              upload.url = Discourse.store.store_upload(f, upload)
              upload.filesize = f.size
              upload.save!(validate: false)
            end
            # remap the URLs
            DbHelper.remap(UrlHelper.absolute(previous_url), upload.url) unless external

            DbHelper.remap(
              previous_url,
              upload.url,
              excluded_tables: %w[
                posts
                post_search_data
                incoming_emails
                notifications
                single_sign_on_records
                stylesheet_cache
                topic_search_data
                users
                user_emails
                draft_sequences
                optimized_images
              ],
            )

            remap_scope ||=
              begin
                Post
                  .with_deleted
                  .where(
                    "raw ~ '/uploads/#{db}/\\d+/' OR raw ~ '/uploads/#{db}/original/(\\d|[a-z])/'",
                  )
                  .select(:id, :raw, :cooked)
                  .all
              end

            remap_scope.each do |post|
              post.raw.gsub!(previous_url, upload.url)
              post.cooked.gsub!(previous_url, upload.url)
              if post.changed?
                Post.with_deleted.where(id: post.id).update_all(raw: post.raw, cooked: post.cooked)
              end
            end

            upload.optimized_images.find_each(&:destroy!)
            upload.rebake_posts_on_old_scheme
            # remove the old file (when local)
            FileUtils.rm(path, force: true) unless external
          rescue => e
            problems << { upload: upload, ex: e }
          ensure
            file&.unlink
            file&.close
          end
        end
      end
    end

    problems
  end

  def self.extract_upload_ids(raw)
    return [] if raw.blank?

    sha1s = []

    raw.scan(/\/(\h{40})/).each { |match| sha1s << match[0] }

    raw
      .scan(%r{/([a-zA-Z0-9]+)})
      .each { |match| sha1s << Upload.sha1_from_base62_encoded(match[0]) }

    Upload.where(sha1: sha1s.uniq).pluck(:id)
  end

  def self.backfill_dominant_colors!(count)
    Upload
      .where(dominant_color: nil)
      .order("id desc")
      .first(count)
      .each { |upload| upload.calculate_dominant_color! }
  end

  private

  def short_url_basename
    "#{Upload.base62_sha1(sha1)}#{extension.present? ? ".#{extension}" : ""}"
  end
end

# == Schema Information
#
# Table name: uploads
#
#  id                           :integer          not null, primary key
#  user_id                      :integer          not null
#  original_filename            :string           not null
#  filesize                     :bigint           not null
#  width                        :integer
#  height                       :integer
#  url                          :string           not null
#  created_at                   :datetime         not null
#  updated_at                   :datetime         not null
#  sha1                         :string(40)
#  origin                       :string(1000)
#  retain_hours                 :integer
#  extension                    :string(10)
#  thumbnail_width              :integer
#  thumbnail_height             :integer
#  etag                         :string
#  secure                       :boolean          default(FALSE), not null
#  access_control_post_id       :bigint
#  original_sha1                :string
#  animated                     :boolean
#  verification_status          :integer          default(1), not null
#  security_last_changed_at     :datetime
#  security_last_changed_reason :string
#  dominant_color               :text
#
# Indexes
#
#  idx_uploads_on_verification_status       (verification_status)
#  index_uploads_on_access_control_post_id  (access_control_post_id)
#  index_uploads_on_etag                    (etag)
#  index_uploads_on_extension               (lower((extension)::text))
#  index_uploads_on_id                      (id) WHERE (dominant_color IS NULL)
#  index_uploads_on_id_and_url              (id,url)
#  index_uploads_on_original_sha1           (original_sha1)
#  index_uploads_on_sha1                    (sha1) UNIQUE
#  index_uploads_on_url                     (url)
#  index_uploads_on_user_id                 (user_id)
#