discourse/spec/models/upload_spec.rb
Martin Brennan 9f275c12ab
FIX: Handle storage providers not implementing ACLs (#13675)
When secure media is enabled or when upload secure status
is updated, we also try and update the upload ACL. However
if the object storage provider does not implement this we
get an Aws::S3::Errors::NotImplemented error. This PR handles
this error so the update_secure_status method does not error
out and still returns whether the secure status changed.
2021-07-09 11:31:44 +10:00

589 lines
20 KiB
Ruby

# frozen_string_literal: true
require 'rails_helper'
describe Upload do
let(:upload) { build(:upload) }
let(:user_id) { 1 }
let(:image_filename) { "logo.png" }
let(:image) { file_from_fixtures(image_filename) }
let(:image_svg_filename) { "image.svg" }
let(:image_svg) { file_from_fixtures(image_svg_filename) }
let(:huge_image_filename) { "huge.jpg" }
let(:huge_image) { file_from_fixtures(huge_image_filename) }
let(:attachment_path) { __FILE__ }
let(:attachment) { File.new(attachment_path) }
describe '.with_no_non_post_relations' do
it "does not find non-post related uploads" do
post_upload = Fabricate(:upload)
post = Fabricate(:post, raw: "<img src='#{post_upload.url}'>")
post.link_post_uploads
badge_upload = Fabricate(:upload)
Fabricate(:badge, image_upload: badge_upload)
avatar_upload = Fabricate(:upload)
Fabricate(:user, uploaded_avatar: avatar_upload)
site_setting_upload = Fabricate(:upload)
SiteSetting.create!(
name: "logo",
data_type: SiteSettings::TypeSupervisor.types[:upload],
value: site_setting_upload.id
)
upload_ids = Upload
.by_users
.with_no_non_post_relations
.pluck(:id)
expect(upload_ids).to eq([post_upload.id])
end
end
context ".create_thumbnail!" do
it "does not create a thumbnail when disabled" do
SiteSetting.create_thumbnails = false
OptimizedImage.expects(:create_for).never
upload.create_thumbnail!(100, 100)
end
it "creates a thumbnail" do
upload = Fabricate(:upload)
thumbnail = Fabricate(:optimized_image, upload: upload)
SiteSetting.expects(:create_thumbnails?).returns(true)
OptimizedImage.expects(:create_for).returns(thumbnail)
upload.create_thumbnail!(100, 100)
upload.reload
expect(upload.optimized_images.count).to eq(1)
end
end
it "supports <style> element in SVG" do
SiteSetting.authorized_extensions = "svg"
upload = UploadCreator.new(image_svg, image_svg_filename).create_for(user_id)
expect(upload.valid?).to eq(true)
path = Discourse.store.path_for(upload)
expect(File.read(path)).to match(/<style>/)
end
it "can reconstruct dimensions on demand" do
upload = UploadCreator.new(huge_image, "image.png").create_for(user_id)
upload.update_columns(width: nil, height: nil, thumbnail_width: nil, thumbnail_height: nil)
upload = Upload.find(upload.id)
expect(upload.width).to eq(8900)
expect(upload.height).to eq(8900)
upload.reload
expect(upload.read_attribute(:width)).to eq(8900)
upload.update_columns(width: nil, height: nil, thumbnail_width: nil, thumbnail_height: nil)
expect(upload.thumbnail_width).to eq(500)
expect(upload.thumbnail_height).to eq(500)
end
it "dimension calculation returns nil on missing image" do
upload = UploadCreator.new(huge_image, "image.png").create_for(user_id)
upload.update_columns(width: nil, height: nil, thumbnail_width: nil, thumbnail_height: nil)
missing_url = "wrong_folder#{upload.url}"
upload.update_columns(url: missing_url)
expect(upload.thumbnail_height).to eq(nil)
expect(upload.thumbnail_width).to eq(nil)
end
it 'returns error when image resolution is to big' do
SiteSetting.max_image_megapixels = 10
upload = UploadCreator.new(huge_image, "image.png").create_for(user_id)
expect(upload.persisted?).to eq(false)
expect(upload.errors.messages[:base].first).to eq(I18n.t("upload.images.larger_than_x_megapixels", max_image_megapixels: 20))
end
it "extracts file extension" do
created_upload = UploadCreator.new(image, image_filename).create_for(user_id)
expect(created_upload.extension).to eq("png")
end
it "should create an invalid upload when the filename is blank" do
SiteSetting.authorized_extensions = "*"
created_upload = UploadCreator.new(attachment, nil).create_for(user_id)
expect(created_upload.valid?).to eq(false)
end
context ".extract_url" do
let(:url) { 'https://example.com/uploads/default/original/1X/d1c2d40ab994e8410c.png' }
it 'should return the right part of url' do
expect(Upload.extract_url(url).to_s).to eq('/original/1X/d1c2d40ab994e8410c.png')
end
end
context ".get_from_url" do
let(:sha1) { "10f73034616a796dfd70177dc54b6def44c4ba6f" }
let(:upload) { Fabricate(:upload, sha1: sha1) }
it "works when the file has been uploaded" do
expect(Upload.get_from_url(upload.url)).to eq(upload)
end
describe 'for an extensionless url' do
before do
upload.update!(url: upload.url.sub('.png', ''))
upload.reload
end
it 'should return the right upload' do
expect(Upload.get_from_url(upload.url)).to eq(upload)
end
end
it "should return the right upload as long as the upload's URL matches" do
upload.update!(url: "/uploads/default/12345/971308e535305c51.png")
expect(Upload.get_from_url(upload.url)).to eq(upload)
expect(Upload.get_from_url("/uploads/default/123131/971308e535305c51.png"))
.to eq(nil)
end
describe 'for a url a tree' do
before do
upload.update!(url:
Discourse.store.get_path_for(
"original",
16001,
upload.sha1,
".#{upload.extension}"
)
)
end
it 'should return the right upload' do
expect(Upload.get_from_url(upload.url)).to eq(upload)
end
end
it "works when using a cdn" do
begin
original_asset_host = Rails.configuration.action_controller.asset_host
Rails.configuration.action_controller.asset_host = 'http://my.cdn.com'
expect(Upload.get_from_url(
URI.join("http://my.cdn.com", upload.url).to_s
)).to eq(upload)
ensure
Rails.configuration.action_controller.asset_host = original_asset_host
end
end
it "should return the right upload when using the full URL" do
expect(Upload.get_from_url(
URI.join("http://discourse.some.com:3000/", upload.url).to_s
)).to eq(upload)
end
it "doesn't blow up with an invalid URI" do
expect { Upload.get_from_url("http://ip:port/index.html") }.not_to raise_error
expect { Upload.get_from_url("mailto:admin%40example.com") }.not_to raise_error
expect { Upload.get_from_url("mailto:example") }.not_to raise_error
end
describe "s3 store" do
let(:upload) { Fabricate(:upload_s3) }
let(:path) { upload.url.sub(SiteSetting.Upload.s3_base_url, '') }
before do
setup_s3
end
it "should return the right upload when using base url (not CDN) for s3" do
upload
expect(Upload.get_from_url(upload.url)).to eq(upload)
end
describe 'when using a cdn' do
let(:s3_cdn_url) { 'https://mycdn.slowly.net' }
before do
SiteSetting.s3_cdn_url = s3_cdn_url
end
it "should return the right upload" do
upload
expect(Upload.get_from_url(URI.join(s3_cdn_url, path).to_s)).to eq(upload)
end
describe 'when upload bucket contains subfolder' do
before do
SiteSetting.s3_upload_bucket = "s3-upload-bucket/path/path2"
end
it "should return the right upload" do
upload
expect(Upload.get_from_url(URI.join(s3_cdn_url, path).to_s)).to eq(upload)
end
end
end
it "should return the right upload when using one CDN for both s3 and assets" do
begin
original_asset_host = Rails.configuration.action_controller.asset_host
cdn_url = 'http://my.cdn.com'
Rails.configuration.action_controller.asset_host = cdn_url
SiteSetting.s3_cdn_url = cdn_url
upload
expect(Upload.get_from_url(
URI.join(cdn_url, path).to_s
)).to eq(upload)
ensure
Rails.configuration.action_controller.asset_host = original_asset_host
end
end
end
end
context ".get_from_urls" do
let(:upload) { Fabricate(:upload, sha1: "10f73034616a796dfd70177dc54b6def44c4ba6f") }
let(:upload2) { Fabricate(:upload, sha1: "2a7081e615f9075befd87a9a6d273935c0262cd5") }
it "works with multiple uploads" do
expect(Upload.get_from_urls([upload.url, upload2.url])).to contain_exactly(upload, upload2)
end
it "works for an extensionless URL" do
url = upload.url.sub('.png', '')
upload.update!(url: url)
expect(Upload.get_from_urls([url])).to contain_exactly(upload)
end
it "works with uploads with mismatched URLs" do
upload.update!(url: "/uploads/default/12345/971308e535305c51.png")
expect(Upload.get_from_urls([upload.url])).to contain_exactly(upload)
expect(Upload.get_from_urls(["/uploads/default/123131/971308e535305c51.png"])).to be_empty
end
it "works with an upload with a URL containing a deep tree" do
upload.update!(url: Discourse.store.get_path_for("original", 16001, upload.sha1, ".#{upload.extension}"))
expect(Upload.get_from_urls([upload.url])).to contain_exactly(upload)
end
it "works when using a CDN" do
begin
original_asset_host = Rails.configuration.action_controller.asset_host
Rails.configuration.action_controller.asset_host = 'http://my.cdn.com'
expect(Upload.get_from_urls([
URI.join("http://my.cdn.com", upload.url).to_s
])).to contain_exactly(upload)
ensure
Rails.configuration.action_controller.asset_host = original_asset_host
end
end
it "works with full URLs" do
expect(Upload.get_from_urls([
URI.join("http://discourse.some.com:3000/", upload.url).to_s
])).to contain_exactly(upload)
end
it "handles invalid URIs" do
urls = ["http://ip:port/index.html", "mailto:admin%40example.com", "mailto:example"]
expect { Upload.get_from_urls(urls) }.not_to raise_error
end
end
describe '.generate_digest' do
it "should return the right digest" do
expect(Upload.generate_digest(image.path)).to eq('bc975735dfc6409c1c2aa5ebf2239949bcbdbd65')
end
end
describe '.short_url' do
it "should generate a correct short url" do
upload = Upload.new(sha1: 'bda2c513e1da04f7b4e99230851ea2aafeb8cc4e', extension: 'png')
expect(upload.short_url).to eq('upload://r3AYqESanERjladb4vBB7VsMBm6.png')
upload.extension = nil
expect(upload.short_url).to eq('upload://r3AYqESanERjladb4vBB7VsMBm6')
end
end
describe '.sha1_from_short_url' do
it "should be able to look up sha1" do
sha1 = 'bda2c513e1da04f7b4e99230851ea2aafeb8cc4e'
expect(Upload.sha1_from_short_url('upload://r3AYqESanERjladb4vBB7VsMBm6.png')).to eq(sha1)
expect(Upload.sha1_from_short_url('upload://r3AYqESanERjladb4vBB7VsMBm6')).to eq(sha1)
expect(Upload.sha1_from_short_url('r3AYqESanERjladb4vBB7VsMBm6')).to eq(sha1)
end
it "should be able to look up sha1 even with leading zeros" do
sha1 = '0000c513e1da04f7b4e99230851ea2aafeb8cc4e'
expect(Upload.sha1_from_short_url('upload://1Eg9p8rrCURq4T3a6iJUk0ri6.png')).to eq(sha1)
end
end
describe '#base62_sha1' do
it 'should return the right value' do
upload.update!(sha1: "0000c513e1da04f7b4e99230851ea2aafeb8cc4e")
expect(upload.base62_sha1).to eq("1Eg9p8rrCURq4T3a6iJUk0ri6")
end
end
describe '.sha1_from_short_path' do
it "should be able to lookup sha1" do
path = "/uploads/short-url/3UjQ4jHoyeoQndk5y3qHzm3QVTQ.png"
sha1 = "1b6453892473a467d07372d45eb05abc2031647a"
expect(Upload.sha1_from_short_path(path)).to eq(sha1)
expect(Upload.sha1_from_short_path(path.sub(".png", ""))).to eq(sha1)
end
end
describe '#to_s' do
it 'should return the right value' do
expect(upload.to_s).to eq(upload.url)
end
end
describe '.migrate_to_new_scheme' do
it 'should not migrate system uploads' do
SiteSetting.migrate_to_new_scheme = true
expect { Upload.migrate_to_new_scheme }
.to_not change { Upload.pluck(:url) }
end
end
describe '.update_secure_status' do
it "respects the override parameter if provided" do
upload.update!(secure: true)
upload.update_secure_status(override: true)
expect(upload.secure).to eq(true)
upload.update_secure_status(override: false)
expect(upload.secure).to eq(false)
end
it 'marks a local upload as not secure with default settings' do
upload.update!(secure: true)
expect { upload.update_secure_status }
.to change { upload.secure }
expect(upload.secure).to eq(false)
end
context "local attachment" do
before do
SiteSetting.authorized_extensions = "pdf"
end
let(:upload) { Fabricate(:upload, original_filename: "small.pdf", extension: "pdf", secure: true) }
it 'marks a local attachment as secure if secure media enabled' do
upload.update!(secure: false, access_control_post: Fabricate(:private_message_post))
enable_secure_media
expect { upload.update_secure_status }
.to change { upload.secure }
expect(upload.secure).to eq(true)
end
it 'marks a local attachment as not secure if secure media enabled' do
expect { upload.update_secure_status }
.to change { upload.secure }
expect(upload.secure).to eq(false)
end
end
it 'does not change secure status of a non-attachment when prevent_anons_from_downloading_files is enabled by itself' do
SiteSetting.prevent_anons_from_downloading_files = true
SiteSetting.authorized_extensions = "mp4"
upload.update!(original_filename: "small.mp4", extension: "mp4")
expect { upload.update_secure_status }
.not_to change { upload.secure }
expect(upload.secure).to eq(false)
end
context "secure media enabled" do
before do
enable_secure_media
end
it 'does not mark an image upload as not secure when there is no access control post id, to avoid unintentional exposure' do
upload.update!(secure: true)
upload.update_secure_status
expect(upload.secure).to eq(true)
end
it 'marks the upload as not secure if its access control post is a public post' do
upload.update!(secure: true, access_control_post: Fabricate(:post))
upload.update_secure_status
expect(upload.secure).to eq(false)
end
it 'leaves the upload as secure if its access control post is a PM post' do
upload.update!(secure: true, access_control_post: Fabricate(:private_message_post))
upload.update_secure_status
expect(upload.secure).to eq(true)
end
it 'marks an image upload as secure if login_required is enabled' do
SiteSetting.login_required = true
upload.update!(secure: false)
expect { upload.update_secure_status }
.to change { upload.secure }
expect(upload.reload.secure).to eq(true)
end
it 'does not mark an upload used for a custom emoji as secure' do
SiteSetting.login_required = true
upload.update!(secure: false)
CustomEmoji.create(name: 'meme', upload: upload)
upload.update_secure_status
expect(upload.reload.secure).to eq(false)
end
it 'does not mark an upload whose origin matches a regular emoji as secure (sometimes emojis are downloaded in pull_hotlinked_images)' do
SiteSetting.login_required = true
falafel = Emoji.all.find { |e| e.url == '/images/emoji/twitter/falafel.png?v=9' }
upload.update!(secure: false, origin: "http://localhost:3000#{falafel.url}")
upload.update_secure_status
expect(upload.reload.secure).to eq(false)
end
it 'does not mark any upload with origin containing images/emoji in the URL' do
SiteSetting.login_required = true
upload.update!(secure: false, origin: "http://localhost:3000/images/emoji/test.png")
upload.update_secure_status
expect(upload.reload.secure).to eq(false)
end
it "does not throw an error if the object storage provider does not support ACLs" do
FileStore::S3Store.any_instance.stubs(:update_upload_ACL).raises(
Aws::S3::Errors::NotImplemented.new("A header you provided implies functionality that is not implemented", "")
)
upload.update!(secure: true, access_control_post: Fabricate(:private_message_post))
expect { upload.update_secure_status }.not_to raise_error
end
end
end
def enable_secure_media
setup_s3
SiteSetting.secure_media = true
stub_upload(upload)
end
context '.destroy' do
it "can correctly clear information when destroying an upload" do
upload = Fabricate(:upload)
user = Fabricate(:user)
user.user_profile.update!(
card_background_upload_id: upload.id,
profile_background_upload_id: upload.id
)
upload.destroy
user.user_profile.reload
expect(user.user_profile.card_background_upload_id).to eq(nil)
expect(user.user_profile.profile_background_upload_id).to eq(nil)
end
end
context ".signed_url_from_secure_media_url" do
before do
# must be done so signed_url_for_path exists
enable_secure_media
end
it "correctly gives back a signed url from a path only" do
secure_url = "/secure-media-uploads/original/1X/c5a2c4ba0fa390f5aac5c2c1a12416791ebdd9e9.png"
signed_url = Upload.signed_url_from_secure_media_url(secure_url)
expect(signed_url).not_to include("secure-media-uploads")
expect(UrlHelper.s3_presigned_url?(signed_url)).to eq(true)
end
it "correctly gives back a signed url from a full url" do
secure_url = "http://localhost:3000/secure-media-uploads/original/1X/c5a2c4ba0fa390f5aac5c2c1a12416791ebdd9e9.png"
signed_url = Upload.signed_url_from_secure_media_url(secure_url)
expect(signed_url).not_to include(Discourse.base_url)
expect(UrlHelper.s3_presigned_url?(signed_url)).to eq(true)
end
end
context ".secure_media_url_from_upload_url" do
before do
# must be done so signed_url_for_path exists
enable_secure_media
end
it "gets the secure media url from an S3 upload url" do
upload = Fabricate(:upload_s3, secure: true)
url = upload.url
secure_url = Upload.secure_media_url_from_upload_url(url)
expect(secure_url).not_to include(SiteSetting.Upload.absolute_base_url)
end
end
context ".secure_media_url?" do
it "works for a secure media url with or without schema + host" do
url = "//localhost:3000/secure-media-uploads/original/2X/f/f62055931bb702c7fd8f552fb901f977e0289a18.png"
expect(Upload.secure_media_url?(url)).to eq(true)
url = "/secure-media-uploads/original/2X/f/f62055931bb702c7fd8f552fb901f977e0289a18.png"
expect(Upload.secure_media_url?(url)).to eq(true)
url = "http://localhost:3000/secure-media-uploads/original/2X/f/f62055931bb702c7fd8f552fb901f977e0289a18.png"
expect(Upload.secure_media_url?(url)).to eq(true)
end
it "does not get false positives on a topic url" do
url = "/t/secure-media-uploads-are-cool/42839"
expect(Upload.secure_media_url?(url)).to eq(false)
end
it "returns true only for secure media URL for actual media (images/video/audio)" do
url = "/secure-media-uploads/original/2X/f/f62055931bb702c7fd8f552fb901f977e0289a18.mp4"
expect(Upload.secure_media_url?(url)).to eq(true)
url = "/secure-media-uploads/original/2X/f/f62055931bb702c7fd8f552fb901f977e0289a18.png"
expect(Upload.secure_media_url?(url)).to eq(true)
url = "/secure-media-uploads/original/2X/f/f62055931bb702c7fd8f552fb901f977e0289a18.mp3"
expect(Upload.secure_media_url?(url)).to eq(true)
url = "/secure-media-uploads/original/2X/f/f62055931bb702c7fd8f552fb901f977e0289a18.pdf"
expect(Upload.secure_media_url?(url)).to eq(false)
end
it "does not work for regular upload urls" do
url = "/uploads/default/test_0/original/1X/e1864389d8252958586c76d747b069e9f68827e3.png"
expect(Upload.secure_media_url?(url)).to eq(false)
end
it "does not raise for invalid URLs" do
url = "http://URL:%20https://google.com"
expect(Upload.secure_media_url?(url)).to eq(false)
end
end
end