discourse/lib/external_upload_helpers.rb

# frozen_string_literal: true

# Extends controllers with the methods required to do direct
# external uploads.
module ExternalUploadHelpers
  extend ActiveSupport::Concern

  class ExternalUploadValidationError < StandardError
  end

  PRESIGNED_PUT_RATE_LIMIT_PER_MINUTE = 10
  CREATE_MULTIPART_RATE_LIMIT_PER_MINUTE = 10
  COMPLETE_MULTIPART_RATE_LIMIT_PER_MINUTE = 10

  included do
    before_action :external_store_check,
                  only: %i[
                    generate_presigned_put
                    complete_external_upload
                    create_multipart
                    batch_presign_multipart_parts
                    abort_multipart
                    complete_multipart
                  ]
    before_action :direct_s3_uploads_check,
                  only: %i[
                    generate_presigned_put
                    complete_external_upload
                    create_multipart
                    batch_presign_multipart_parts
                    abort_multipart
                    complete_multipart
                  ]
    before_action :can_upload_external?, only: %i[create_multipart generate_presigned_put]
  end

  def generate_presigned_put
    RateLimiter.new(
      current_user,
      "generate-presigned-put-upload-stub",
      ExternalUploadHelpers::PRESIGNED_PUT_RATE_LIMIT_PER_MINUTE,
      1.minute,
    ).performed!

    file_name = params.require(:file_name)
    file_size = params.require(:file_size).to_i
    type = params.require(:type)

    begin
      validate_before_create_direct_upload(
        file_name: file_name,
        file_size: file_size,
        upload_type: type,
      )
    rescue ExternalUploadValidationError => err
      return render_json_error(err.message, status: 422)
    end

    external_upload_data =
      ExternalUploadManager.create_direct_upload(
        current_user: current_user,
        file_name: file_name,
        file_size: file_size,
        upload_type: type,
        metadata: parse_allowed_metadata(params[:metadata]),
      )

    render json: external_upload_data
  end

  def complete_external_upload
    unique_identifier = params.require(:unique_identifier)
    external_upload_stub =
      ExternalUploadStub.find_by(unique_identifier: unique_identifier, created_by: current_user)
    return render_404 if external_upload_stub.blank?

    complete_external_upload_via_manager(external_upload_stub)
  end

  def create_multipart
    RateLimiter.new(
      current_user,
      "create-multipart-upload",
      ExternalUploadHelpers::CREATE_MULTIPART_RATE_LIMIT_PER_MINUTE,
      1.minute,
    ).performed!

    file_name = params.require(:file_name)
    file_size = params.require(:file_size).to_i
    upload_type = params.require(:upload_type)

    begin
      validate_before_create_multipart(
        file_name: file_name,
        file_size: file_size,
        upload_type: upload_type,
      )
    rescue ExternalUploadValidationError => err
      return render_json_error(err.message, status: 422)
    end

    begin
      external_upload_data =
        create_direct_multipart_upload do
          ExternalUploadManager.create_direct_multipart_upload(
            current_user: current_user,
            file_name: file_name,
            file_size: file_size,
            upload_type: upload_type,
            metadata: parse_allowed_metadata(params[:metadata]),
          )
        end
    rescue ExternalUploadHelpers::ExternalUploadValidationError => err
      return render_json_error(err.message, status: 422)
    end

    render json: external_upload_data
  end

  def batch_presign_multipart_parts
    part_numbers = params.require(:part_numbers)
    unique_identifier = params.require(:unique_identifier)

    ##
    # NOTE: This is configurable by hidden site setting because this really is heavily
    # dependent on upload speed. We request 5-10 URLs at a time with this endpoint; for
    # a 1.5GB upload with 5mb parts this could mean 60 requests to the server to get all
    # the part URLs. If the user's upload speed is super fast they may request all 60
    # batches in a minute, if it is slow they may request 5 batches in a minute.
    #
    # The other external upload endpoints are not hit as often, so they can stay as constant
    # values for now.
    RateLimiter.new(
      current_user,
      "batch-presign",
      SiteSetting.max_batch_presign_multipart_per_minute,
      1.minute,
    ).performed!

    part_numbers = part_numbers.map { |part_number| validate_part_number(part_number) }

    external_upload_stub =
      ExternalUploadStub.find_by(unique_identifier: unique_identifier, created_by: current_user)
    return render_404 if external_upload_stub.blank?

    return render_404 if !multipart_upload_exists?(external_upload_stub)

    store = multipart_store(external_upload_stub.upload_type)

    presigned_urls = {}
    part_numbers.each do |part_number|
      presigned_urls[part_number] = store.presign_multipart_part(
        upload_id: external_upload_stub.external_upload_identifier,
        key: external_upload_stub.key,
        part_number: part_number,
      )
    end

    render json: { presigned_urls: presigned_urls }
  end

  def multipart_upload_exists?(external_upload_stub)
    store = multipart_store(external_upload_stub.upload_type)
    begin
      store.list_multipart_parts(
        upload_id: external_upload_stub.external_upload_identifier,
        key: external_upload_stub.key,
        max_parts: 1,
      )
    rescue Aws::S3::Errors::NoSuchUpload => err
      debug_upload_error(
        err,
        I18n.t(
          "upload.external_upload_not_found",
          additional_detail: "path: #{external_upload_stub.key}",
        ),
      )
      return false
    end
    true
  end

  def abort_multipart
    external_upload_identifier = params.require(:external_upload_identifier)
    external_upload_stub =
      ExternalUploadStub.find_by(external_upload_identifier: external_upload_identifier)

    # The stub could have already been deleted by an earlier error via
    # ExternalUploadManager, so we consider this a great success if the
    # stub is already gone.
    return render json: success_json if external_upload_stub.blank?

    return render_404 if external_upload_stub.created_by_id != current_user.id
    store = multipart_store(external_upload_stub.upload_type)

    begin
      store.abort_multipart(
        upload_id: external_upload_stub.external_upload_identifier,
        key: external_upload_stub.key,
      )
    rescue Aws::S3::Errors::ServiceError => err
      return(
        render_json_error(
          debug_upload_error(
            err,
            I18n.t(
              "upload.abort_multipart_failure",
              additional_detail: "external upload stub id: #{external_upload_stub.id}",
            ),
          ),
          status: 422,
        )
      )
    end

    external_upload_stub.destroy!

    render json: success_json
  end

  def complete_multipart
    unique_identifier = params.require(:unique_identifier)
    parts = params.require(:parts)

    RateLimiter.new(
      current_user,
      "complete-multipart-upload",
      ExternalUploadHelpers::COMPLETE_MULTIPART_RATE_LIMIT_PER_MINUTE,
      1.minute,
    ).performed!

    external_upload_stub =
      ExternalUploadStub.find_by(unique_identifier: unique_identifier, created_by: current_user)
    return render_404 if external_upload_stub.blank?

    return render_404 if !multipart_upload_exists?(external_upload_stub)

    store = multipart_store(external_upload_stub.upload_type)
    parts =
      parts
        .map do |part|
          part_number = part[:part_number]
          etag = part[:etag]
          part_number = validate_part_number(part_number)

          if etag.blank?
            raise Discourse::InvalidParameters.new(
                    "All parts must have an etag and a valid part number",
                  )
          end

          # this is done so it's an array of hashes rather than an array of
          # ActionController::Parameters
          { part_number: part_number, etag: etag }
        end
        .sort_by { |part| part[:part_number] }

    begin
      complete_response =
        store.complete_multipart(
          upload_id: external_upload_stub.external_upload_identifier,
          key: external_upload_stub.key,
          parts: parts,
        )
    rescue Aws::S3::Errors::ServiceError => err
      return(
        render_json_error(
          debug_upload_error(
            err,
            I18n.t(
              "upload.complete_multipart_failure",
              additional_detail: "external upload stub id: #{external_upload_stub.id}",
            ),
          ),
          status: 422,
        )
      )
    end

    complete_external_upload_via_manager(external_upload_stub)
  end

  private

  def complete_external_upload_via_manager(external_upload_stub)
    opts = {
      for_private_message: params[:for_private_message]&.to_s == "true",
      for_site_setting: params[:for_site_setting]&.to_s == "true",
      pasted: params[:pasted]&.to_s == "true",
    }

    external_upload_manager = ExternalUploadManager.new(external_upload_stub, opts)
    hijack do
      begin
        upload = external_upload_manager.transform!

        if upload.errors.empty?
          response_serialized = self.class.serialize_upload(upload)
          external_upload_stub.destroy!
          render json: response_serialized, status: 200
        else
          render_json_error(upload.errors.to_hash.values.flatten, status: 422)
        end
      rescue ExternalUploadManager::SizeMismatchError => err
        render_json_error(
          debug_upload_error(
            err,
            I18n.t("upload.size_mismatch_failure", additional_detail: err.message),
          ),
          status: 422,
        )
      rescue ExternalUploadManager::ChecksumMismatchError => err
        render_json_error(
          debug_upload_error(
            err,
            I18n.t("upload.checksum_mismatch_failure", additional_detail: err.message),
          ),
          status: 422,
        )
      rescue ExternalUploadManager::CannotPromoteError => err
        render_json_error(
          debug_upload_error(
            err,
            I18n.t("upload.cannot_promote_failure", additional_detail: err.message),
          ),
          status: 422,
        )
      rescue ExternalUploadManager::DownloadFailedError, Aws::S3::Errors::NotFound => err
        render_json_error(
          debug_upload_error(
            err,
            I18n.t("upload.download_failure", additional_detail: err.message),
          ),
          status: 422,
        )
      rescue => err
        Discourse.warn_exception(
          err,
          message: "Complete external upload failed unexpectedly for user #{current_user.id}",
        )

        render_json_error(I18n.t("upload.failed"), status: 422)
      end
    end
  end

  def validate_before_create_direct_upload(file_name:, file_size:, upload_type:)
    # noop, should be overridden
  end

  def validate_before_create_multipart(file_name:, file_size:, upload_type:)
    # noop, should be overridden
  end

  def validate_part_number(part_number)
    part_number = part_number.to_i
    if !part_number.between?(1, 10_000)
      raise Discourse::InvalidParameters.new("Each part number should be between 1 and 10000")
    end
    part_number
  end

  def debug_upload_error(err, friendly_message)
    return if !SiteSetting.enable_upload_debug_mode
    Discourse.warn_exception(err, message: friendly_message)
    Rails.env.development? ? friendly_message : I18n.t("upload.failed")
  end

  def multipart_store(upload_type)
    ExternalUploadManager.store_for_upload_type(upload_type)
  end

  def external_store_check
    return render_404 if !Discourse.store.external?
  end

  def direct_s3_uploads_check
    return render_404 if !SiteSetting.enable_direct_s3_uploads
  end

  def can_upload_external?
    raise Discourse::InvalidAccess if !guardian.can_upload_external?
  end

  # don't want people posting arbitrary S3 metadata so we just take the
  # one we need. all of these will be converted to x-amz-meta- metadata
  # fields in S3 so it's best to use dashes in the names for consistency
  #
  # this metadata is baked into the presigned url and is not altered when
  # sending the PUT from the clientside to the presigned url
  def parse_allowed_metadata(metadata)
    return if metadata.blank?
    metadata.permit("sha1-checksum").to_h
  end

  def render_404
    raise Discourse::NotFound
  end
end