2019-05-03 06:17:27 +08:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2017-06-13 19:27:05 +08:00
|
|
|
require "mini_mime"
|
2017-05-11 06:16:57 +08:00
|
|
|
|
2013-02-06 03:16:51 +08:00
|
|
|
class UploadsController < ApplicationController
|
2021-11-18 07:17:23 +08:00
|
|
|
include ExternalUploadHelpers
|
2024-02-20 09:19:22 +08:00
|
|
|
include SecureUploadEndpointHelpers
|
2021-11-18 07:17:23 +08:00
|
|
|
|
2022-09-29 07:24:33 +08:00
|
|
|
requires_login except: %i[show show_short _show_secure_deprecated show_secure]
|
2018-02-01 12:17:59 +08:00
|
|
|
|
2022-09-29 07:24:33 +08:00
|
|
|
skip_before_action :preload_json,
|
|
|
|
:check_xhr,
|
|
|
|
:redirect_to_login_if_required,
|
2024-06-25 19:32:18 +08:00
|
|
|
:redirect_to_profile_if_required,
|
2022-09-29 07:24:33 +08:00
|
|
|
only: %i[show show_short _show_secure_deprecated show_secure]
|
2019-10-14 12:40:33 +08:00
|
|
|
protect_from_forgery except: :show
|
2013-04-03 07:17:17 +08:00
|
|
|
|
2022-09-29 07:24:33 +08:00
|
|
|
before_action :is_asset_path,
|
|
|
|
:apply_cdn_headers,
|
|
|
|
only: %i[show show_short _show_secure_deprecated show_secure]
|
|
|
|
before_action :external_store_check, only: %i[_show_secure_deprecated show_secure]
|
2019-11-18 13:56:20 +08:00
|
|
|
|
2020-05-18 22:00:41 +08:00
|
|
|
SECURE_REDIRECT_GRACE_SECONDS = 5
|
|
|
|
|
2013-02-06 03:16:51 +08:00
|
|
|
def create
|
2017-11-23 14:28:18 +08:00
|
|
|
# capture current user for block later on
|
|
|
|
me = current_user
|
2024-02-07 12:55:36 +08:00
|
|
|
RateLimiter.new(
|
|
|
|
current_user,
|
|
|
|
"uploads-per-minute",
|
|
|
|
SiteSetting.max_uploads_per_minute,
|
|
|
|
1.minute.to_i,
|
|
|
|
).performed!
|
|
|
|
|
2021-07-13 10:22:00 +08:00
|
|
|
params.permit(:type, :upload_type)
|
|
|
|
raise Discourse::InvalidParameters if params[:type].blank? && params[:upload_type].blank?
|
2017-05-18 18:13:13 +08:00
|
|
|
# 50 characters ought to be enough for the upload type
|
2021-07-13 10:22:00 +08:00
|
|
|
type =
|
|
|
|
(params[:upload_type].presence || params[:type].presence).parameterize(separator: "_")[0..50]
|
2023-01-09 20:20:10 +08:00
|
|
|
|
2023-12-13 07:53:19 +08:00
|
|
|
if type == "avatar" &&
|
2021-08-24 15:46:28 +08:00
|
|
|
(
|
|
|
|
SiteSetting.discourse_connect_overrides_avatar ||
|
2023-12-13 07:53:19 +08:00
|
|
|
!me.in_any_groups?(SiteSetting.uploaded_avatars_allowed_groups_map)
|
2023-01-09 20:20:10 +08:00
|
|
|
)
|
2017-05-11 06:16:57 +08:00
|
|
|
return render json: failed_json, status: 422
|
2015-11-12 17:26:45 +08:00
|
|
|
end
|
|
|
|
|
2017-06-23 18:13:48 +08:00
|
|
|
url = params[:url]
|
|
|
|
file = params[:file] || params[:files]&.first
|
|
|
|
pasted = params[:pasted] == "true"
|
|
|
|
for_private_message = params[:for_private_message] == "true"
|
2018-11-14 15:03:02 +08:00
|
|
|
for_site_setting = params[:for_site_setting] == "true"
|
2017-11-27 09:43:18 +08:00
|
|
|
is_api = is_api?
|
|
|
|
retain_hours = params[:retain_hours].to_i
|
|
|
|
|
|
|
|
# note, atm hijack is processed in its own context and has not access to controller
|
|
|
|
# longer term we may change this
|
|
|
|
hijack do
|
2017-12-27 23:33:25 +08:00
|
|
|
begin
|
|
|
|
info =
|
|
|
|
UploadsController.create_upload(
|
|
|
|
current_user: me,
|
|
|
|
file: file,
|
|
|
|
url: url,
|
|
|
|
type: type,
|
|
|
|
for_private_message: for_private_message,
|
2018-11-14 15:03:02 +08:00
|
|
|
for_site_setting: for_site_setting,
|
2017-12-27 23:33:25 +08:00
|
|
|
pasted: pasted,
|
|
|
|
is_api: is_api,
|
|
|
|
retain_hours: retain_hours,
|
|
|
|
)
|
|
|
|
rescue => e
|
|
|
|
render json: failed_json.merge(message: e.message&.split("\n")&.first), status: 422
|
|
|
|
else
|
|
|
|
render json: UploadsController.serialize_upload(info), status: Upload === info ? 200 : 422
|
|
|
|
end
|
2014-09-23 13:50:26 +08:00
|
|
|
end
|
2013-02-06 03:16:51 +08:00
|
|
|
end
|
2013-06-05 06:34:53 +08:00
|
|
|
|
2017-08-23 04:40:01 +08:00
|
|
|
def lookup_urls
|
|
|
|
params.permit(short_urls: [])
|
|
|
|
uploads = []
|
|
|
|
|
|
|
|
if (params[:short_urls] && params[:short_urls].length > 0)
|
2019-05-29 09:00:25 +08:00
|
|
|
PrettyText::Helpers
|
|
|
|
.lookup_upload_urls(params[:short_urls])
|
|
|
|
.each do |short_url, paths|
|
|
|
|
uploads << { short_url: short_url, url: paths[:url], short_path: paths[:short_path] }
|
2017-08-23 04:40:01 +08:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
render json: uploads.to_json
|
|
|
|
end
|
|
|
|
|
2013-09-07 01:18:42 +08:00
|
|
|
def show
|
2019-06-26 22:02:55 +08:00
|
|
|
# do not serve uploads requested via XHR to prevent XSS
|
2019-06-27 17:13:44 +08:00
|
|
|
return xhr_not_allowed if request.xhr?
|
2019-06-26 22:02:55 +08:00
|
|
|
|
2014-05-14 08:51:09 +08:00
|
|
|
return render_404 if !RailsMultisite::ConnectionManagement.has_db?(params[:site])
|
|
|
|
|
2014-03-25 07:37:31 +08:00
|
|
|
RailsMultisite::ConnectionManagement.with_connection(params[:site]) do |db|
|
2014-09-10 00:40:11 +08:00
|
|
|
return render_404 if SiteSetting.prevent_anons_from_downloading_files && current_user.nil?
|
2013-09-07 01:18:42 +08:00
|
|
|
|
2015-05-20 21:32:31 +08:00
|
|
|
if upload =
|
|
|
|
Upload.find_by(sha1: params[:sha]) ||
|
|
|
|
Upload.find_by(id: params[:id], url: request.env["PATH_INFO"])
|
2019-05-15 04:36:54 +08:00
|
|
|
unless Discourse.store.internal?
|
|
|
|
local_store = FileStore::LocalStore.new
|
|
|
|
return render_404 unless local_store.has_been_uploaded?(upload.url)
|
|
|
|
end
|
|
|
|
|
2019-05-29 09:00:25 +08:00
|
|
|
send_file_local_upload(upload)
|
|
|
|
else
|
|
|
|
render_404
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
2019-01-30 05:47:25 +08:00
|
|
|
|
2019-05-29 09:00:25 +08:00
|
|
|
def show_short
|
2019-06-26 22:02:55 +08:00
|
|
|
# do not serve uploads requested via XHR to prevent XSS
|
2019-06-27 17:13:44 +08:00
|
|
|
return xhr_not_allowed if request.xhr?
|
2019-06-26 22:02:55 +08:00
|
|
|
|
2019-05-29 09:00:25 +08:00
|
|
|
return render_404 if SiteSetting.prevent_anons_from_downloading_files && current_user.nil?
|
2019-05-28 23:18:21 +08:00
|
|
|
|
2019-05-29 09:00:25 +08:00
|
|
|
sha1 = Upload.sha1_from_base62_encoded(params[:base62])
|
|
|
|
|
|
|
|
if upload = Upload.find_by(sha1: sha1)
|
2022-09-29 07:24:33 +08:00
|
|
|
return handle_secure_upload_request(upload) if upload.secure? && SiteSetting.secure_uploads?
|
2020-01-16 11:50:27 +08:00
|
|
|
|
2019-05-29 09:00:25 +08:00
|
|
|
if Discourse.store.internal?
|
|
|
|
send_file_local_upload(upload)
|
2014-04-15 04:55:57 +08:00
|
|
|
else
|
2022-03-21 22:28:52 +08:00
|
|
|
redirect_to Discourse.store.url_for(upload, force_download: force_download?),
|
|
|
|
allow_other_host: true
|
2014-04-15 04:55:57 +08:00
|
|
|
end
|
2019-05-29 09:00:25 +08:00
|
|
|
else
|
|
|
|
render_404
|
2014-03-25 07:37:31 +08:00
|
|
|
end
|
2013-09-07 01:18:42 +08:00
|
|
|
end
|
|
|
|
|
2022-09-29 07:24:33 +08:00
|
|
|
# Kept to avoid rebaking old posts with /show-secure-uploads/ in their
|
|
|
|
# contents, this will ensure the uploads in these posts continue to
|
|
|
|
# work in future.
|
|
|
|
def _show_secure_deprecated
|
|
|
|
show_secure
|
|
|
|
end
|
|
|
|
|
2019-11-18 09:25:42 +08:00
|
|
|
def show_secure
|
|
|
|
# do not serve uploads requested via XHR to prevent XSS
|
|
|
|
return xhr_not_allowed if request.xhr?
|
2020-01-07 10:27:24 +08:00
|
|
|
|
|
|
|
path_with_ext = "#{params[:path]}.#{params[:extension]}"
|
2024-02-20 09:19:22 +08:00
|
|
|
upload = upload_from_path_and_extension(path_with_ext)
|
2020-01-07 10:27:24 +08:00
|
|
|
|
|
|
|
return render_404 if upload.blank?
|
2019-11-18 09:25:42 +08:00
|
|
|
|
2020-03-26 05:16:02 +08:00
|
|
|
return render_404 if SiteSetting.prevent_anons_from_downloading_files && current_user.nil?
|
2022-09-29 07:24:33 +08:00
|
|
|
return handle_secure_upload_request(upload, path_with_ext) if SiteSetting.secure_uploads?
|
2020-01-07 10:27:24 +08:00
|
|
|
|
2022-09-29 07:24:33 +08:00
|
|
|
# we don't want to 404 here if secure uploads gets disabled
|
2020-01-07 10:27:24 +08:00
|
|
|
# because all posts with secure uploads will show broken media
|
|
|
|
# until rebaked, which could take some time
|
2020-01-07 12:02:17 +08:00
|
|
|
#
|
|
|
|
# if the upload is still secure, that means the ACL is probably still
|
|
|
|
# private, so we don't want to go to the CDN url just yet otherwise we
|
|
|
|
# will get a 403. if the upload is not secure we assume the ACL is public
|
2020-03-16 09:54:14 +08:00
|
|
|
signed_secure_url = Discourse.store.signed_url_for_path(path_with_ext)
|
2022-03-21 22:28:52 +08:00
|
|
|
redirect_to upload.secure? ? signed_secure_url : Discourse.store.cdn_url(upload.url),
|
|
|
|
allow_other_host: true
|
2019-11-18 09:25:42 +08:00
|
|
|
end
|
|
|
|
|
2020-03-16 09:54:14 +08:00
|
|
|
def handle_secure_upload_request(upload, path_with_ext = nil)
|
2024-02-20 09:19:22 +08:00
|
|
|
check_secure_upload_permission(upload)
|
2020-01-16 11:50:27 +08:00
|
|
|
|
2020-07-03 11:42:36 +08:00
|
|
|
# defaults to public: false, so only cached by the client browser
|
2022-05-26 07:53:01 +08:00
|
|
|
cache_seconds =
|
|
|
|
SiteSetting.s3_presigned_get_url_expires_after_seconds - SECURE_REDIRECT_GRACE_SECONDS
|
2020-07-03 11:42:36 +08:00
|
|
|
expires_in cache_seconds.seconds
|
2020-05-18 22:00:41 +08:00
|
|
|
|
2020-03-16 09:54:14 +08:00
|
|
|
# url_for figures out the full URL, handling multisite DBs,
|
|
|
|
# and will return a presigned URL for the upload
|
|
|
|
if path_with_ext.blank?
|
2022-03-21 22:28:52 +08:00
|
|
|
return(
|
|
|
|
redirect_to Discourse.store.url_for(upload, force_download: force_download?),
|
|
|
|
allow_other_host: true
|
2023-01-09 20:20:10 +08:00
|
|
|
)
|
2020-03-16 09:54:14 +08:00
|
|
|
end
|
|
|
|
|
2020-07-03 11:42:36 +08:00
|
|
|
redirect_to Discourse.store.signed_url_for_path(
|
2020-09-29 10:12:03 +08:00
|
|
|
path_with_ext,
|
2022-05-26 07:53:01 +08:00
|
|
|
expires_in: SiteSetting.s3_presigned_get_url_expires_after_seconds,
|
2020-09-29 10:12:03 +08:00
|
|
|
force_download: force_download?,
|
2022-03-21 22:28:52 +08:00
|
|
|
),
|
|
|
|
allow_other_host: true
|
2020-01-16 11:50:27 +08:00
|
|
|
end
|
|
|
|
|
2019-02-21 10:13:37 +08:00
|
|
|
def metadata
|
|
|
|
params.require(:url)
|
|
|
|
upload = Upload.get_from_url(params[:url])
|
|
|
|
raise Discourse::NotFound unless upload
|
|
|
|
|
|
|
|
render json: {
|
|
|
|
original_filename: upload.original_filename,
|
|
|
|
width: upload.width,
|
|
|
|
height: upload.height,
|
|
|
|
human_filesize: upload.human_filesize,
|
|
|
|
}
|
|
|
|
end
|
|
|
|
|
2021-11-18 07:17:23 +08:00
|
|
|
protected
|
2021-07-28 06:42:25 +08:00
|
|
|
|
2021-11-18 07:17:23 +08:00
|
|
|
def validate_before_create_multipart(file_name:, file_size:, upload_type:)
|
|
|
|
validate_file_size(file_name: file_name, file_size: file_size)
|
FEATURE: Uppy direct S3 multipart uploads in composer (#14051)
This pull request introduces the endpoints required, and the JavaScript functionality in the `ComposerUppyUpload` mixin, for direct S3 multipart uploads. There are four new endpoints in the uploads controller:
* `create-multipart.json` - Creates the multipart upload in S3 along with an `ExternalUploadStub` record, storing information about the file in the same way as `generate-presigned-put.json` does for regular direct S3 uploads
* `batch-presign-multipart-parts.json` - Takes a list of part numbers and the unique identifier for an `ExternalUploadStub` record, and generates the presigned URLs for those parts if the multipart upload still exists and if the user has permission to access that upload
* `complete-multipart.json` - Completes the multipart upload in S3. Needs the full list of part numbers and their associated ETags which are returned when the part is uploaded to the presigned URL above. Only works if the user has permission to access the associated `ExternalUploadStub` record and the multipart upload still exists.
After we confirm the upload is complete in S3, we go through the regular `UploadCreator` flow, the same as `complete-external-upload.json`, and promote the temporary upload S3 into a full `Upload` record, moving it to its final destination.
* `abort-multipart.json` - Aborts the multipart upload on S3 and destroys the `ExternalUploadStub` record if the user has permission to access that upload.
Also added are a few new columns to `ExternalUploadStub`:
* multipart - Whether or not this is a multipart upload
* external_upload_identifier - The "upload ID" for an S3 multipart upload
* filesize - The size of the file when the `create-multipart.json` or `generate-presigned-put.json` is called. This is used for validation.
When the user completes a direct S3 upload, either regular or multipart, we take the `filesize` that was captured when the `ExternalUploadStub` was first created and compare it with the final `Content-Length` size of the file where it is stored in S3. Then, if the two do not match, we throw an error, delete the file on S3, and ban the user from uploading files for N (default 5) minutes. This would only happen if the user uploads a different file than what they first specified, or in the case of multipart uploads uploaded larger chunks than needed. This is done to prevent abuse of S3 storage by bad actors.
Also included in this PR is an update to vendor/uppy.js. This has been built locally from the latest uppy source at https://github.com/transloadit/uppy/commit/d613b849a6591083f8a0968aa8d66537e231bbcd. This must be done so that I can get my multipart upload changes into Discourse. When the Uppy team cuts a proper release, we can bump the package.json versions instead.
2021-08-25 06:46:54 +08:00
|
|
|
end
|
2021-07-28 06:42:25 +08:00
|
|
|
|
2021-11-18 07:17:23 +08:00
|
|
|
def validate_before_create_direct_upload(file_name:, file_size:, upload_type:)
|
|
|
|
validate_file_size(file_name: file_name, file_size: file_size)
|
2021-07-28 06:42:25 +08:00
|
|
|
end
|
|
|
|
|
2021-11-18 07:17:23 +08:00
|
|
|
def validate_file_size(file_name:, file_size:)
|
2022-03-07 10:39:33 +08:00
|
|
|
raise ExternalUploadValidationError.new(I18n.t("upload.size_zero_failure")) if file_size.zero?
|
|
|
|
|
2023-05-18 16:36:34 +08:00
|
|
|
if attachment_too_big?(file_name, file_size)
|
2021-11-18 07:17:23 +08:00
|
|
|
raise ExternalUploadValidationError.new(
|
|
|
|
I18n.t(
|
|
|
|
"upload.attachments.too_large_humanized",
|
|
|
|
max_size:
|
|
|
|
ActiveSupport::NumberHelper.number_to_human_size(
|
2024-08-16 22:03:39 +08:00
|
|
|
UploadsController.max_attachment_size_for_user(current_user).kilobytes,
|
FEATURE: Direct S3 multipart uploads for backups (#14736)
This PR introduces a new `enable_experimental_backup_uploads` site setting (default false and hidden), which when enabled alongside `enable_direct_s3_uploads` will allow for direct S3 multipart uploads of backup .tar.gz files.
To make multipart external uploads work with both the S3BackupStore and the S3Store, I've had to move several methods out of S3Store and into S3Helper, including:
* presigned_url
* create_multipart
* abort_multipart
* complete_multipart
* presign_multipart_part
* list_multipart_parts
Then, S3Store and S3BackupStore either delegate directly to S3Helper or have their own special methods to call S3Helper for these methods. FileStore.temporary_upload_path has also removed its dependence on upload_path, and can now be used interchangeably between the stores. A similar change was made in the frontend as well, moving the multipart related JS code out of ComposerUppyUpload and into a mixin of its own, so it can also be used by UppyUploadMixin.
Some changes to ExternalUploadManager had to be made here as well. The backup direct uploads do not need an Upload record made for them in the database, so they can be moved to their final S3 resting place when completing the multipart upload.
This changeset is not perfect; it introduces some special cases in UploadController to handle backups that was previously in BackupController, because UploadController is where the multipart routes are located. A subsequent pull request will pull these routes into a module or some other sharing pattern, along with hooks, so the backup controller and the upload controller (and any future controllers that may need them) can include these routes in a nicer way.
2021-11-11 06:25:31 +08:00
|
|
|
),
|
2023-01-09 20:20:10 +08:00
|
|
|
),
|
FEATURE: Direct S3 multipart uploads for backups (#14736)
This PR introduces a new `enable_experimental_backup_uploads` site setting (default false and hidden), which when enabled alongside `enable_direct_s3_uploads` will allow for direct S3 multipart uploads of backup .tar.gz files.
To make multipart external uploads work with both the S3BackupStore and the S3Store, I've had to move several methods out of S3Store and into S3Helper, including:
* presigned_url
* create_multipart
* abort_multipart
* complete_multipart
* presign_multipart_part
* list_multipart_parts
Then, S3Store and S3BackupStore either delegate directly to S3Helper or have their own special methods to call S3Helper for these methods. FileStore.temporary_upload_path has also removed its dependence on upload_path, and can now be used interchangeably between the stores. A similar change was made in the frontend as well, moving the multipart related JS code out of ComposerUppyUpload and into a mixin of its own, so it can also be used by UppyUploadMixin.
Some changes to ExternalUploadManager had to be made here as well. The backup direct uploads do not need an Upload record made for them in the database, so they can be moved to their final S3 resting place when completing the multipart upload.
This changeset is not perfect; it introduces some special cases in UploadController to handle backups that was previously in BackupController, because UploadController is where the multipart routes are located. A subsequent pull request will pull these routes into a module or some other sharing pattern, along with hooks, so the backup controller and the upload controller (and any future controllers that may need them) can include these routes in a nicer way.
2021-11-11 06:25:31 +08:00
|
|
|
)
|
FEATURE: Uppy direct S3 multipart uploads in composer (#14051)
This pull request introduces the endpoints required, and the JavaScript functionality in the `ComposerUppyUpload` mixin, for direct S3 multipart uploads. There are four new endpoints in the uploads controller:
* `create-multipart.json` - Creates the multipart upload in S3 along with an `ExternalUploadStub` record, storing information about the file in the same way as `generate-presigned-put.json` does for regular direct S3 uploads
* `batch-presign-multipart-parts.json` - Takes a list of part numbers and the unique identifier for an `ExternalUploadStub` record, and generates the presigned URLs for those parts if the multipart upload still exists and if the user has permission to access that upload
* `complete-multipart.json` - Completes the multipart upload in S3. Needs the full list of part numbers and their associated ETags which are returned when the part is uploaded to the presigned URL above. Only works if the user has permission to access the associated `ExternalUploadStub` record and the multipart upload still exists.
After we confirm the upload is complete in S3, we go through the regular `UploadCreator` flow, the same as `complete-external-upload.json`, and promote the temporary upload S3 into a full `Upload` record, moving it to its final destination.
* `abort-multipart.json` - Aborts the multipart upload on S3 and destroys the `ExternalUploadStub` record if the user has permission to access that upload.
Also added are a few new columns to `ExternalUploadStub`:
* multipart - Whether or not this is a multipart upload
* external_upload_identifier - The "upload ID" for an S3 multipart upload
* filesize - The size of the file when the `create-multipart.json` or `generate-presigned-put.json` is called. This is used for validation.
When the user completes a direct S3 upload, either regular or multipart, we take the `filesize` that was captured when the `ExternalUploadStub` was first created and compare it with the final `Content-Length` size of the file where it is stored in S3. Then, if the two do not match, we throw an error, delete the file on S3, and ban the user from uploading files for N (default 5) minutes. This would only happen if the user uploads a different file than what they first specified, or in the case of multipart uploads uploaded larger chunks than needed. This is done to prevent abuse of S3 storage by bad actors.
Also included in this PR is an update to vendor/uppy.js. This has been built locally from the latest uppy source at https://github.com/transloadit/uppy/commit/d613b849a6591083f8a0968aa8d66537e231bbcd. This must be done so that I can get my multipart upload changes into Discourse. When the Uppy team cuts a proper release, we can bump the package.json versions instead.
2021-08-25 06:46:54 +08:00
|
|
|
end
|
2023-05-18 16:36:34 +08:00
|
|
|
|
|
|
|
if image_too_big?(file_name, file_size)
|
|
|
|
raise ExternalUploadValidationError.new(
|
|
|
|
I18n.t(
|
|
|
|
"upload.images.too_large_humanized",
|
|
|
|
max_size:
|
|
|
|
ActiveSupport::NumberHelper.number_to_human_size(
|
|
|
|
SiteSetting.max_image_size_kb.kilobytes,
|
|
|
|
),
|
|
|
|
),
|
|
|
|
)
|
|
|
|
end
|
FEATURE: Direct S3 multipart uploads for backups (#14736)
This PR introduces a new `enable_experimental_backup_uploads` site setting (default false and hidden), which when enabled alongside `enable_direct_s3_uploads` will allow for direct S3 multipart uploads of backup .tar.gz files.
To make multipart external uploads work with both the S3BackupStore and the S3Store, I've had to move several methods out of S3Store and into S3Helper, including:
* presigned_url
* create_multipart
* abort_multipart
* complete_multipart
* presign_multipart_part
* list_multipart_parts
Then, S3Store and S3BackupStore either delegate directly to S3Helper or have their own special methods to call S3Helper for these methods. FileStore.temporary_upload_path has also removed its dependence on upload_path, and can now be used interchangeably between the stores. A similar change was made in the frontend as well, moving the multipart related JS code out of ComposerUppyUpload and into a mixin of its own, so it can also be used by UppyUploadMixin.
Some changes to ExternalUploadManager had to be made here as well. The backup direct uploads do not need an Upload record made for them in the database, so they can be moved to their final S3 resting place when completing the multipart upload.
This changeset is not perfect; it introduces some special cases in UploadController to handle backups that was previously in BackupController, because UploadController is where the multipart routes are located. A subsequent pull request will pull these routes into a module or some other sharing pattern, along with hooks, so the backup controller and the upload controller (and any future controllers that may need them) can include these routes in a nicer way.
2021-11-11 06:25:31 +08:00
|
|
|
end
|
|
|
|
|
2020-09-29 10:12:03 +08:00
|
|
|
def force_download?
|
|
|
|
params[:dl] == "1"
|
|
|
|
end
|
|
|
|
|
2019-06-27 17:13:44 +08:00
|
|
|
def xhr_not_allowed
|
|
|
|
raise Discourse::InvalidParameters.new("XHR not allowed")
|
|
|
|
end
|
|
|
|
|
2017-11-27 09:43:18 +08:00
|
|
|
def self.serialize_upload(data)
|
2017-08-23 04:40:01 +08:00
|
|
|
# as_json.as_json is not a typo... as_json in AM serializer returns keys as symbols, we need them
|
|
|
|
# as strings here
|
|
|
|
serialized = UploadSerializer.new(data, root: nil).as_json.as_json if Upload === data
|
|
|
|
serialized ||= (data || {}).as_json
|
|
|
|
end
|
|
|
|
|
2018-11-14 15:03:02 +08:00
|
|
|
def self.create_upload(
|
|
|
|
current_user:,
|
|
|
|
file:,
|
|
|
|
url:,
|
|
|
|
type:,
|
|
|
|
for_private_message:,
|
|
|
|
for_site_setting:,
|
|
|
|
pasted:,
|
|
|
|
is_api:,
|
|
|
|
retain_hours:
|
|
|
|
)
|
2017-05-11 06:16:57 +08:00
|
|
|
if file.nil?
|
2017-11-27 09:43:18 +08:00
|
|
|
if url.present? && is_api
|
2017-05-11 06:16:57 +08:00
|
|
|
maximum_upload_size = [
|
|
|
|
SiteSetting.max_image_size_kb,
|
2024-08-16 22:03:39 +08:00
|
|
|
UploadsController.max_attachment_size_for_user(current_user),
|
2017-05-11 06:16:57 +08:00
|
|
|
].max.kilobytes
|
2017-05-25 01:42:52 +08:00
|
|
|
tempfile =
|
2023-01-09 20:20:10 +08:00
|
|
|
begin
|
2017-05-25 01:42:52 +08:00
|
|
|
FileHelper.download(
|
|
|
|
url,
|
2019-05-28 08:28:57 +08:00
|
|
|
follow_redirect: true,
|
2017-05-25 01:42:52 +08:00
|
|
|
max_file_size: maximum_upload_size,
|
|
|
|
tmp_file_name: "discourse-upload-#{type}",
|
|
|
|
)
|
|
|
|
rescue StandardError
|
|
|
|
nil
|
2023-01-09 20:20:10 +08:00
|
|
|
end
|
2017-05-11 06:16:57 +08:00
|
|
|
filename = File.basename(URI.parse(url).path)
|
2015-08-13 00:33:13 +08:00
|
|
|
end
|
2017-05-11 06:16:57 +08:00
|
|
|
else
|
|
|
|
tempfile = file.tempfile
|
|
|
|
filename = file.original_filename
|
|
|
|
end
|
2015-08-13 00:33:13 +08:00
|
|
|
|
2017-05-11 06:16:57 +08:00
|
|
|
return { errors: [I18n.t("upload.file_missing")] } if tempfile.nil?
|
2015-06-15 22:12:15 +08:00
|
|
|
|
2017-06-23 18:13:48 +08:00
|
|
|
opts = {
|
|
|
|
type: type,
|
|
|
|
for_private_message: for_private_message,
|
2018-11-14 15:03:02 +08:00
|
|
|
for_site_setting: for_site_setting,
|
2017-06-23 18:13:48 +08:00
|
|
|
pasted: pasted,
|
|
|
|
}
|
2017-06-13 04:41:29 +08:00
|
|
|
|
|
|
|
upload = UploadCreator.new(tempfile, filename, opts).create_for(current_user.id)
|
2015-06-15 22:12:15 +08:00
|
|
|
|
2017-05-11 06:16:57 +08:00
|
|
|
if upload.errors.empty? && current_user.admin?
|
|
|
|
upload.update_columns(retain_hours: retain_hours) if retain_hours > 0
|
2015-06-15 22:12:15 +08:00
|
|
|
end
|
|
|
|
|
2019-04-30 14:58:18 +08:00
|
|
|
upload.errors.empty? ? upload : { errors: upload.errors.to_hash.values.flatten }
|
2017-05-11 06:16:57 +08:00
|
|
|
ensure
|
2018-03-28 16:20:08 +08:00
|
|
|
tempfile&.close!
|
2016-06-20 18:35:07 +08:00
|
|
|
end
|
|
|
|
|
2019-05-29 09:00:25 +08:00
|
|
|
private
|
|
|
|
|
2024-08-16 22:03:39 +08:00
|
|
|
def self.max_attachment_size_for_user(user)
|
|
|
|
if user.id == Discourse::SYSTEM_USER_ID && !SiteSetting.system_user_max_attachment_size_kb.zero?
|
|
|
|
SiteSetting.system_user_max_attachment_size_kb
|
|
|
|
else
|
|
|
|
SiteSetting.max_attachment_size_kb
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2023-05-18 16:36:34 +08:00
|
|
|
# We can preemptively check size for attachments, but not for (most) images
|
FEATURE: Uppy direct S3 multipart uploads in composer (#14051)
This pull request introduces the endpoints required, and the JavaScript functionality in the `ComposerUppyUpload` mixin, for direct S3 multipart uploads. There are four new endpoints in the uploads controller:
* `create-multipart.json` - Creates the multipart upload in S3 along with an `ExternalUploadStub` record, storing information about the file in the same way as `generate-presigned-put.json` does for regular direct S3 uploads
* `batch-presign-multipart-parts.json` - Takes a list of part numbers and the unique identifier for an `ExternalUploadStub` record, and generates the presigned URLs for those parts if the multipart upload still exists and if the user has permission to access that upload
* `complete-multipart.json` - Completes the multipart upload in S3. Needs the full list of part numbers and their associated ETags which are returned when the part is uploaded to the presigned URL above. Only works if the user has permission to access the associated `ExternalUploadStub` record and the multipart upload still exists.
After we confirm the upload is complete in S3, we go through the regular `UploadCreator` flow, the same as `complete-external-upload.json`, and promote the temporary upload S3 into a full `Upload` record, moving it to its final destination.
* `abort-multipart.json` - Aborts the multipart upload on S3 and destroys the `ExternalUploadStub` record if the user has permission to access that upload.
Also added are a few new columns to `ExternalUploadStub`:
* multipart - Whether or not this is a multipart upload
* external_upload_identifier - The "upload ID" for an S3 multipart upload
* filesize - The size of the file when the `create-multipart.json` or `generate-presigned-put.json` is called. This is used for validation.
When the user completes a direct S3 upload, either regular or multipart, we take the `filesize` that was captured when the `ExternalUploadStub` was first created and compare it with the final `Content-Length` size of the file where it is stored in S3. Then, if the two do not match, we throw an error, delete the file on S3, and ban the user from uploading files for N (default 5) minutes. This would only happen if the user uploads a different file than what they first specified, or in the case of multipart uploads uploaded larger chunks than needed. This is done to prevent abuse of S3 storage by bad actors.
Also included in this PR is an update to vendor/uppy.js. This has been built locally from the latest uppy source at https://github.com/transloadit/uppy/commit/d613b849a6591083f8a0968aa8d66537e231bbcd. This must be done so that I can get my multipart upload changes into Discourse. When the Uppy team cuts a proper release, we can bump the package.json versions instead.
2021-08-25 06:46:54 +08:00
|
|
|
# as they may be further reduced in size by UploadCreator (at this point
|
|
|
|
# they may have already been reduced in size by preprocessors)
|
2023-05-18 16:36:34 +08:00
|
|
|
def attachment_too_big?(file_name, file_size)
|
FEATURE: Uppy direct S3 multipart uploads in composer (#14051)
This pull request introduces the endpoints required, and the JavaScript functionality in the `ComposerUppyUpload` mixin, for direct S3 multipart uploads. There are four new endpoints in the uploads controller:
* `create-multipart.json` - Creates the multipart upload in S3 along with an `ExternalUploadStub` record, storing information about the file in the same way as `generate-presigned-put.json` does for regular direct S3 uploads
* `batch-presign-multipart-parts.json` - Takes a list of part numbers and the unique identifier for an `ExternalUploadStub` record, and generates the presigned URLs for those parts if the multipart upload still exists and if the user has permission to access that upload
* `complete-multipart.json` - Completes the multipart upload in S3. Needs the full list of part numbers and their associated ETags which are returned when the part is uploaded to the presigned URL above. Only works if the user has permission to access the associated `ExternalUploadStub` record and the multipart upload still exists.
After we confirm the upload is complete in S3, we go through the regular `UploadCreator` flow, the same as `complete-external-upload.json`, and promote the temporary upload S3 into a full `Upload` record, moving it to its final destination.
* `abort-multipart.json` - Aborts the multipart upload on S3 and destroys the `ExternalUploadStub` record if the user has permission to access that upload.
Also added are a few new columns to `ExternalUploadStub`:
* multipart - Whether or not this is a multipart upload
* external_upload_identifier - The "upload ID" for an S3 multipart upload
* filesize - The size of the file when the `create-multipart.json` or `generate-presigned-put.json` is called. This is used for validation.
When the user completes a direct S3 upload, either regular or multipart, we take the `filesize` that was captured when the `ExternalUploadStub` was first created and compare it with the final `Content-Length` size of the file where it is stored in S3. Then, if the two do not match, we throw an error, delete the file on S3, and ban the user from uploading files for N (default 5) minutes. This would only happen if the user uploads a different file than what they first specified, or in the case of multipart uploads uploaded larger chunks than needed. This is done to prevent abuse of S3 storage by bad actors.
Also included in this PR is an update to vendor/uppy.js. This has been built locally from the latest uppy source at https://github.com/transloadit/uppy/commit/d613b849a6591083f8a0968aa8d66537e231bbcd. This must be done so that I can get my multipart upload changes into Discourse. When the Uppy team cuts a proper release, we can bump the package.json versions instead.
2021-08-25 06:46:54 +08:00
|
|
|
!FileHelper.is_supported_image?(file_name) &&
|
2024-08-16 22:03:39 +08:00
|
|
|
file_size >= UploadsController.max_attachment_size_for_user(current_user).kilobytes
|
FEATURE: Uppy direct S3 multipart uploads in composer (#14051)
This pull request introduces the endpoints required, and the JavaScript functionality in the `ComposerUppyUpload` mixin, for direct S3 multipart uploads. There are four new endpoints in the uploads controller:
* `create-multipart.json` - Creates the multipart upload in S3 along with an `ExternalUploadStub` record, storing information about the file in the same way as `generate-presigned-put.json` does for regular direct S3 uploads
* `batch-presign-multipart-parts.json` - Takes a list of part numbers and the unique identifier for an `ExternalUploadStub` record, and generates the presigned URLs for those parts if the multipart upload still exists and if the user has permission to access that upload
* `complete-multipart.json` - Completes the multipart upload in S3. Needs the full list of part numbers and their associated ETags which are returned when the part is uploaded to the presigned URL above. Only works if the user has permission to access the associated `ExternalUploadStub` record and the multipart upload still exists.
After we confirm the upload is complete in S3, we go through the regular `UploadCreator` flow, the same as `complete-external-upload.json`, and promote the temporary upload S3 into a full `Upload` record, moving it to its final destination.
* `abort-multipart.json` - Aborts the multipart upload on S3 and destroys the `ExternalUploadStub` record if the user has permission to access that upload.
Also added are a few new columns to `ExternalUploadStub`:
* multipart - Whether or not this is a multipart upload
* external_upload_identifier - The "upload ID" for an S3 multipart upload
* filesize - The size of the file when the `create-multipart.json` or `generate-presigned-put.json` is called. This is used for validation.
When the user completes a direct S3 upload, either regular or multipart, we take the `filesize` that was captured when the `ExternalUploadStub` was first created and compare it with the final `Content-Length` size of the file where it is stored in S3. Then, if the two do not match, we throw an error, delete the file on S3, and ban the user from uploading files for N (default 5) minutes. This would only happen if the user uploads a different file than what they first specified, or in the case of multipart uploads uploaded larger chunks than needed. This is done to prevent abuse of S3 storage by bad actors.
Also included in this PR is an update to vendor/uppy.js. This has been built locally from the latest uppy source at https://github.com/transloadit/uppy/commit/d613b849a6591083f8a0968aa8d66537e231bbcd. This must be done so that I can get my multipart upload changes into Discourse. When the Uppy team cuts a proper release, we can bump the package.json versions instead.
2021-08-25 06:46:54 +08:00
|
|
|
end
|
|
|
|
|
2023-05-18 16:36:34 +08:00
|
|
|
# Gifs are not resized on the client and not reduced in size by UploadCreator
|
|
|
|
def image_too_big?(file_name, file_size)
|
|
|
|
FileHelper.is_supported_image?(file_name) && File.extname(file_name) == ".gif" &&
|
|
|
|
file_size >= SiteSetting.max_image_size_kb.kilobytes
|
|
|
|
end
|
|
|
|
|
2019-05-29 09:00:25 +08:00
|
|
|
def send_file_local_upload(upload)
|
|
|
|
opts = {
|
|
|
|
filename: upload.original_filename,
|
|
|
|
content_type: MiniMime.lookup_by_filename(upload.original_filename)&.content_type,
|
|
|
|
}
|
|
|
|
|
2020-07-09 11:31:48 +08:00
|
|
|
if !FileHelper.is_inline_image?(upload.original_filename)
|
2019-05-29 09:00:25 +08:00
|
|
|
opts[:disposition] = "attachment"
|
2019-12-11 21:21:41 +08:00
|
|
|
elsif params[:inline]
|
|
|
|
opts[:disposition] = "inline"
|
2019-05-29 09:00:25 +08:00
|
|
|
end
|
|
|
|
|
|
|
|
file_path = Discourse.store.path_for(upload)
|
|
|
|
return render_404 unless file_path
|
|
|
|
|
|
|
|
send_file(file_path, opts)
|
|
|
|
end
|
|
|
|
|
2021-11-18 07:17:23 +08:00
|
|
|
def create_direct_multipart_upload
|
|
|
|
begin
|
|
|
|
yield
|
|
|
|
rescue Aws::S3::Errors::ServiceError => err
|
|
|
|
message =
|
|
|
|
debug_upload_error(
|
|
|
|
err,
|
|
|
|
I18n.t("upload.create_multipart_failure", additional_detail: err.message),
|
|
|
|
)
|
|
|
|
raise ExternalUploadHelpers::ExternalUploadValidationError.new(message)
|
|
|
|
end
|
FEATURE: Direct S3 multipart uploads for backups (#14736)
This PR introduces a new `enable_experimental_backup_uploads` site setting (default false and hidden), which when enabled alongside `enable_direct_s3_uploads` will allow for direct S3 multipart uploads of backup .tar.gz files.
To make multipart external uploads work with both the S3BackupStore and the S3Store, I've had to move several methods out of S3Store and into S3Helper, including:
* presigned_url
* create_multipart
* abort_multipart
* complete_multipart
* presign_multipart_part
* list_multipart_parts
Then, S3Store and S3BackupStore either delegate directly to S3Helper or have their own special methods to call S3Helper for these methods. FileStore.temporary_upload_path has also removed its dependence on upload_path, and can now be used interchangeably between the stores. A similar change was made in the frontend as well, moving the multipart related JS code out of ComposerUppyUpload and into a mixin of its own, so it can also be used by UppyUploadMixin.
Some changes to ExternalUploadManager had to be made here as well. The backup direct uploads do not need an Upload record made for them in the database, so they can be moved to their final S3 resting place when completing the multipart upload.
This changeset is not perfect; it introduces some special cases in UploadController to handle backups that was previously in BackupController, because UploadController is where the multipart routes are located. A subsequent pull request will pull these routes into a module or some other sharing pattern, along with hooks, so the backup controller and the upload controller (and any future controllers that may need them) can include these routes in a nicer way.
2021-11-11 06:25:31 +08:00
|
|
|
end
|
2013-02-06 03:16:51 +08:00
|
|
|
end
|